fix: build better-sqlite3 (#27968)

* fix: build better-sqlite3 * fix syntax * fix syntax * fix syntax * fix syntax * fix syntax * fix syntax * fix syntax * fix syntax * better sqlite3 * publish pre-release * Update CHANGELOG.md * add comment * try to cache * try and speed things up even more * fixes * improve caching * improve caching * improve caching * fix build * fix build * Update scripts/binary/build.ts Co-authored-by: Adam Stone-Lord <adams@cypress.io> * Update centos7-builder.Dockerfile Co-authored-by: Adam Stone-Lord <adams@cypress.io> * Update centos7-builder.Dockerfile Co-authored-by: Adam Stone-Lord <adams@cypress.io> * Update CHANGELOG.md * fix keys * Update cli/CHANGELOG.md * Update cli/CHANGELOG.md * Revert "fix keys" This reverts commit b1383d4432. * fix keys * fix build * optimize * bump cache * optimize * bump cache * blank * blank * Update scripts/binary/trigger-publish-binary-pipeline.js * Update cli/CHANGELOG.md * Update CHANGELOG.md * Update cli/CHANGELOG.md --------- Co-authored-by: Adam Stone-Lord <adams@cypress.io>
2026-04-26 00:50:41 -05:00 · 2023-10-09 14:23:56 -05:00
parent 1a6f87922c
commit 9f3d5b4565
7 changed files with 129 additions and 29 deletions
@@ -1,3 +1,3 @@
 # Bump this version to force CI to re-create the cache from scratch.

-09-22-23
+10-05-23
@@ -30,8 +30,7 @@ mainBuildFilters: &mainBuildFilters
        - /^release\/\d+\.\d+\.\d+$/
        # use the following branch as well to ensure that v8 snapshot cache updates are fully tested
        - 'publish-binary'
-        - 'chore/update_electron25_and_node18'
-        - 'lerna-optimize-tasks'
+        - 'ryanm/fix/better-sqlite3'

 # usually we don't build Mac app - it takes a long time
 # but sometimes we want to really confirm we are doing the right thing
@@ -42,8 +41,7 @@ macWorkflowFilters: &darwin-workflow-filters
    - equal: [ develop, << pipeline.git.branch >> ]
    # use the following branch as well to ensure that v8 snapshot cache updates are fully tested
    - equal: [ 'update-v8-snapshot-cache-on-develop', << pipeline.git.branch >> ]
-    - equal: [ 'chore/update_webpack_deps_to_latest_webpack4_compat', << pipeline.git.branch >> ]
-    - equal: [ 'lerna-optimize-tasks', << pipeline.git.branch >> ]
+    - equal: [ 'ryanm/fix/better-sqlite3', << pipeline.git.branch >> ]
    - matches:
        pattern: /^release\/\d+\.\d+\.\d+$/
        value: << pipeline.git.branch >>
@@ -54,9 +52,7 @@ linuxArm64WorkflowFilters: &linux-arm64-workflow-filters
    - equal: [ develop, << pipeline.git.branch >> ]
    # use the following branch as well to ensure that v8 snapshot cache updates are fully tested
    - equal: [ 'update-v8-snapshot-cache-on-develop', << pipeline.git.branch >> ]
-    - equal: [ 'chore/update_webpack_deps_to_latest_webpack4_compat', << pipeline.git.branch >> ]
-    - equal: [ 'chore/bump_loaders_and_optimize_webpack', << pipeline.git.branch >> ]
-    - equal: [ 'lerna-optimize-tasks', << pipeline.git.branch >> ]
+    - equal: [ 'ryanm/fix/better-sqlite3', << pipeline.git.branch >> ]
    - matches:
        pattern: /^release\/\d+\.\d+\.\d+$/
        value: << pipeline.git.branch >>
@@ -79,8 +75,7 @@ windowsWorkflowFilters: &windows-workflow-filters
    - equal: [ develop, << pipeline.git.branch >> ]
    # use the following branch as well to ensure that v8 snapshot cache updates are fully tested
    - equal: [ 'update-v8-snapshot-cache-on-develop', << pipeline.git.branch >> ]
-    - equal: [ 'chore/update_webpack_deps_to_latest_webpack4_compat', << pipeline.git.branch >> ]
-    - equal: [ 'lerna-optimize-tasks', << pipeline.git.branch >> ]
+    - equal: [ 'ryanm/fix/better-sqlite3', << pipeline.git.branch >> ]
    - matches:
        pattern: /^release\/\d+\.\d+\.\d+$/
        value: << pipeline.git.branch >>
@@ -150,7 +145,7 @@ commands:
          name: Set environment variable to determine whether or not to persist artifacts
          command: |
            echo "Setting SHOULD_PERSIST_ARTIFACTS variable"
-            echo 'if ! [[ "$CIRCLE_BRANCH" != "develop" && "$CIRCLE_BRANCH" != "release/"* && "$CIRCLE_BRANCH" != "publish-binary" && "$CIRCLE_BRANCH" != "lerna-optimize-tasks" ]]; then
+            echo 'if ! [[ "$CIRCLE_BRANCH" != "develop" && "$CIRCLE_BRANCH" != "release/"* && "$CIRCLE_BRANCH" != "publish-binary" && "$CIRCLE_BRANCH" != "update-v8-snapshot-cache-on-develop" && "$CIRCLE_BRANCH" != "ryanm/fix/better-sqlite3" ]]; then
                export SHOULD_PERSIST_ARTIFACTS=true
            fi' >> "$BASH_ENV"
  # You must run `setup_should_persist_artifacts` command and be using bash before running this command
@@ -345,6 +340,9 @@ commands:
      only-cache-for-root-user:
        type: boolean
        default: false
+      build-better-sqlite3:
+        type: boolean
+        default: false
    steps:
      - install_cache_helpers_dependencies
      - run:
@@ -353,9 +351,18 @@ commands:
      - run:
          name: Generate platform key
          command: node ./scripts/get-platform-key.js > platform_key
-      - restore_cache:
-          name: Restore cache state, to check for known modules cache existence
-          key: v{{ checksum ".circleci/cache-version.txt" }}-{{ checksum "platform_key" }}-state-of-node-modules-cache-{{ checksum "circle_cache_key" }}
+      - when:
+          condition: <<parameters.build-better-sqlite3>>
+          steps:
+            - restore_cache:
+                name: Restore cache state, to check for known modules cache existence
+                key: v{{ checksum ".circleci/cache-version.txt" }}-{{ checksum "platform_key" }}-state-of-node-modules-cache-{{ checksum "circle_cache_key" }}-{{ checksum "centos7-builder.Dockerfile" }}
+      - unless:
+          condition: <<parameters.build-better-sqlite3>>
+          steps:
+            - restore_cache:
+                name: Restore cache state, to check for known modules cache existence
+                key: v{{ checksum ".circleci/cache-version.txt" }}-{{ checksum "platform_key" }}-state-of-node-modules-cache-{{ checksum "circle_cache_key" }}
      - run:
          name: Bail if cache exists
          command: |
@@ -377,6 +384,10 @@ commands:
            PERCY_POSTINSTALL_BROWSER=true \
            yarn --prefer-offline --frozen-lockfile --cache-folder ~/.yarn
          no_output_timeout: 20m
+      - when:
+          condition: <<parameters.build-better-sqlite3>>
+          steps:
+            - build-better-sqlite3
      - prepare-modules-cache:
          dont-move: <<parameters.only-cache-for-root-user>> # we don't move, so we don't hit any issues unpacking symlinks
      - when:
@@ -399,11 +410,22 @@ commands:
                paths:
                  - /tmp/node_modules_cache
      - run: touch node_modules_installed
-      - save_cache:
-          name: Saving node-modules cache state key
-          key: v{{ checksum ".circleci/cache-version.txt" }}-{{ checksum "platform_key" }}-state-of-node-modules-cache-{{ checksum "circle_cache_key" }}
-          paths:
-            - node_modules_installed
+      - when:
+          condition: <<parameters.build-better-sqlite3>>
+          steps:
+            - save_cache:
+                name: Saving node-modules cache state key
+                key: v{{ checksum ".circleci/cache-version.txt" }}-{{ checksum "platform_key" }}-state-of-node-modules-cache-{{ checksum "circle_cache_key" }}-{{ checksum "centos7-builder.Dockerfile" }}
+                paths:
+                  - node_modules_installed
+      - unless:
+          condition: <<parameters.build-better-sqlite3>>
+          steps:
+            - save_cache:
+                name: Saving node-modules cache state key
+                key: v{{ checksum ".circleci/cache-version.txt" }}-{{ checksum "platform_key" }}-state-of-node-modules-cache-{{ checksum "circle_cache_key" }}
+                paths:
+                  - node_modules_installed
      - save_cache:
          name: Save weekly yarn cache
          key: v{{ checksum ".circleci/cache-version.txt" }}-{{ checksum "platform_key" }}-deps-root-weekly-{{ checksum "cache_date" }}
@@ -488,6 +510,63 @@ commands:
            which google-chrome-<<parameters.channel>> || (printf "\n\033[0;31mChrome was not successfully downloaded - bailing\033[0m\n\n" && exit 1)
            echo "Location of Google Chrome Installation: `which google-chrome-<<parameters.channel>>`"
            echo "Google Chrome Version: `google-chrome-<<parameters.channel>> --version`"
+  
+  # This code builds better-sqlite3 on CentOS 7. This is necessary because CentOS 7 has the oldest glibc version
+  # that we support. The script checks for the existence of the Centos7-builder image tar file, and skips if it already
+  # exists. If you want to rebuild the image, set the REBUILD_CENTOS_BUILDER_IMAGE environment variable to any value.
+  # Since this is running Docker remote, we need to copy the project into the container, and copy the built plugin out
+  # of the container because the host running docker does not have access to the project directory so volume mounts are
+  # not possible. The built plugin is copied to the project directory so it can be injected into the final binary.  
+  build-better-sqlite3:
+    description: Build better-sqlite3 for CentOS 7
+    steps:
+      - setup_remote_docker
+      - restore_cache:
+          keys:
+            - cypress-centos7-builder-{{ checksum "centos7-builder.Dockerfile" }}
+      - restore_cache:
+          keys:
+            - better-sqlite3-{{ checksum "node_modules/better-sqlite3/package.json" }}-{{ checksum "node_modules/electron/package.json" }}-{{ checksum "centos7-builder.Dockerfile" }}
+      - run:
+          name: Build or load centos7-builder image
+          command: |
+            if [[ ! -f better_sqlite3.node ]]; then
+              set -x
+              apt update && apt install -y docker.io
+              if [[ ! -f centos7-builder.tar || -n $REBUILD_CENTOS_BUILDER_IMAGE ]]; then
+                  echo "*" > .dockerignore
+                  docker build -t centos7-builder -f centos7-builder.Dockerfile .
+                  docker save centos7-builder > centos7-builder.tar
+                  rm .dockerignore
+              else
+                  docker load < centos7-builder.tar
+              fi
+            fi
+      - save_cache:
+          key: cypress-centos7-builder-{{ checksum "centos7-builder.Dockerfile" }}
+          paths:
+            - centos7-builder.tar
+      - run:
+          name: Build better-sqlite3 for CentOS 7
+          command: |
+            if [[ ! -f better_sqlite3.node ]]; then
+              docker run -d --name centos7-builder centos7-builder /bin/bash -c "sleep 1000000000"
+              docker cp ~/cypress/node_modules/better-sqlite3 centos7-builder:/better-sqlite3
+              docker exec -it centos7-builder /bin/bash -c "cd /better-sqlite3 && source /root/.bashrc && chown -R root:root . && npm install --ignore-scripts && npx --no-install prebuild -r electron -t 25.0.0 --include-regex 'better_sqlite3.node$'"
+              docker cp centos7-builder:/better-sqlite3/build/Release/better_sqlite3.node ~/cypress/node_modules/better-sqlite3/build/Release/better_sqlite3.node
+              docker rm -f centos7-builder
+              cp ~/cypress/node_modules/better-sqlite3/build/Release/better_sqlite3.node ~/cypress/better_sqlite3.node
+            else
+              cp ~/cypress/better_sqlite3.node ~/cypress/node_modules/better-sqlite3/build/Release/better_sqlite3.node
+            fi
+      - save_cache:
+          key: better-sqlite3-{{ checksum "node_modules/better-sqlite3/package.json" }}-{{ checksum "node_modules/electron/package.json" }}-{{ checksum "centos7-builder.Dockerfile" }}
+          paths:
+            - better_sqlite3.node
+      - run:
+          name: Clean up top level better-sqlite3 file
+          command: |
+            rm ~/cypress/better_sqlite3.node

  run-driver-integration-tests:
    parameters:
@@ -1253,6 +1332,9 @@ jobs:
      resource_class:
        type: string
        default: medium
+      build-better-sqlite3:
+        type: boolean
+        default: false
    resource_class: << parameters.resource_class >>
    steps:
      - checkout
@@ -1267,6 +1349,7 @@ jobs:
            - ProgramData/nvm # windows
      - caching-dependency-installer:
          only-cache-for-root-user: <<parameters.only-cache-for-root-user>>
+          build-better-sqlite3: <<parameters.build-better-sqlite3>>
      - store-npm-logs

  ## restores node_modules from previous step & builds if first step skipped
@@ -2518,7 +2601,8 @@ jobs:

 linux-x64-workflow: &linux-x64-workflow
  jobs:
-    - node_modules_install
+    - node_modules_install:
+        build-better-sqlite3: true
    - build:
        context: test-runner:env-canary
        requires:
@@ -0,0 +1,9 @@
+FROM centos:7
+# Install dependencies for re-building better-sqlite and setting devtoolset-8 as the default compiler
+RUN yum -y install centos-release-scl curl python3 make atk-devel atk java-atk-wrapper at-spi2-atk gtk3 libXt libdrm mesa-libgbm Xvfb && yum -y install devtoolset-8-gcc devtoolset-8-gcc-c++
+RUN echo >> /etc/profile.d/devtoolset-8.sh 'source scl_source enable devtoolset-8'
+RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.5/install.sh | bash
+RUN echo >> /etc/profile.d/nvm.sh 'source ~/.nvm/nvm.sh'
+# Node 16 is the most recent version that supports CentOS 7. We only need it to
+# re-build better-sqlite, so there should be minimal risk of security issues.
+RUN source ~/.nvm/nvm.sh && nvm install 16.20.2 && npm install -g yarn
@@ -8,6 +8,7 @@ _Released 10/03/2023 (PENDING)_
 - Fixed an issue where requests were correlated in the wrong order in the proxy. This could cause an issue where the wrong request is used for `cy.intercept` or assets (e.g. stylesheets or images) may not properly be available in Test Replay. Addressed in [#27892](https://github.com/cypress-io/cypress/pull/27892).
 - Fixed an issue where a crashed Chrome renderer can cause the Test Replay recorder to hang. Addressed in [#27909](https://github.com/cypress-io/cypress/pull/27909).
 - Fixed an issue where multiple responses yielded from calls to `cy.wait()` would sometimes be out of order. Fixes [#27337](https://github.com/cypress-io/cypress/issues/27337).
+- Fixed an issue where Test Replay couldn't record tests due to issues involving `GLIBC`. Fixed deprecation warnings during the rebuild of better-sqlite3. Fixes [#27891](https://github.com/cypress-io/cypress/issues/27891) and [#27902](https://github.com/cypress-io/cypress/issues/27902).
 - Enables test replay for executed specs in runs that have a spec that causes a browser crash. Addressed in [#27786](https://github.com/cypress-io/cypress/pull/27786)

 ## 13.3.0
@@ -39,7 +39,7 @@
    "ansi_up": "5.0.0",
    "ast-types": "0.13.3",
    "base64url": "^3.0.1",
-    "better-sqlite3": "8.2.0",
+    "better-sqlite3": "8.7.0",
    "black-hole-stream": "0.0.1",
    "bluebird": "3.7.2",
    "bundle-require": "3.0.4",
@@ -138,8 +138,8 @@ export async function buildCypressApp (options: BuildCypressAppOpts) {
  fs.writeJsonSync(meta.distDir('package.json'), {
    ...packageJsonContents,
    scripts: {
-      // After the `yarn --production` install, we need to patch packages and trigger a server build to rebuild native bindings for `better-sqlite3`
-      postinstall: 'patch-package && yarn workspace @packages/server rebuild-better-sqlite3',
+      // After the `yarn --production` install, we need to patch packages
+      postinstall: 'patch-package',
    },
  }, { spaces: 2 })

@@ -158,6 +158,12 @@ export async function buildCypressApp (options: BuildCypressAppOpts) {
    stdio: 'inherit',
  })

+  log('#copying better-sqlite3')
+  fs.copySync(
+    path.join(CY_ROOT_DIR, 'node_modules', 'better-sqlite3', 'build', 'Release', 'better_sqlite3.node'),
+    path.join(DIST_DIR, 'node_modules', 'better-sqlite3', 'build', 'Release', 'better_sqlite3.node'),
+  )
+
  // TODO: Validate no-hoists / single copies of libs

  // Remove extra directories that are large/unneeded
@@ -9327,13 +9327,13 @@ before-after-hook@^2.2.0:
  resolved "https://registry.yarnpkg.com/before-after-hook/-/before-after-hook-2.2.2.tgz#a6e8ca41028d90ee2c24222f201c90956091613e"
  integrity sha512-3pZEU3NT5BFUo/AD5ERPWOgQOCZITni6iavr5AUw5AUwQjMlI0kzu5btnyD39AF0gUEsDPwJT+oY1ORBJijPjQ==

-better-sqlite3@8.2.0:
-  version "8.2.0"
-  resolved "https://registry.yarnpkg.com/better-sqlite3/-/better-sqlite3-8.2.0.tgz#4ef6185b88992723de7e00cfa67585ac59f320bd"
-  integrity sha512-8eTzxGk9535SB3oSNu0tQ6I4ZffjVCBUjKHN9QeeIFtphBX0sEd0NxAuglBNR9TO5ThnxBB7GqzfcYo9kjadJQ==
+better-sqlite3@8.7.0:
+  version "8.7.0"
+  resolved "https://registry.yarnpkg.com/better-sqlite3/-/better-sqlite3-8.7.0.tgz#bcc341856187b1d110a8a47234fa89c48c8ef538"
+  integrity sha512-99jZU4le+f3G6aIl6PmmV0cxUIWqKieHxsiF7G34CVFiE+/UabpYqkU0NJIkY/96mQKikHeBjtR27vFfs5JpEw==
  dependencies:
    bindings "^1.5.0"
-    prebuild-install "^7.1.0"
+    prebuild-install "^7.1.1"

 big-integer@^1.6.16:
  version "1.6.51"
@@ -23990,7 +23990,7 @@ prebuild-install@^5.2.4, prebuild-install@^5.3.5:
    tunnel-agent "^0.6.0"
    which-pm-runs "^1.0.0"

-prebuild-install@^7.0.0, prebuild-install@^7.1.0:
+prebuild-install@^7.0.0, prebuild-install@^7.1.1:
  version "7.1.1"
  resolved "https://registry.yarnpkg.com/prebuild-install/-/prebuild-install-7.1.1.tgz#de97d5b34a70a0c81334fd24641f2a1702352e45"
  integrity sha512-jAXscXWMcCK8GgCoHOfIr0ODh5ai8mj63L2nWrjuAgXE6tDyYGnx4/8o/rCgU+B4JSyZBKbeZqzhtwtC3ovxjw==