feat: prerendering data for app load (#18704)

* feat: prerendering data for app load

* Fix deps

* Fix create-cypress-tests in build

autobarrel.json

@@ -5,6 +5,7 @@
     "packages/data-context/src/**/*"
   ],
   "ignore": [
+    "packages/data-context/src/gen",
     "packages/graphql/src/stitching",
     "packages/graphql/src/testing",
     "packages/graphql/src/gen"

graphql-codegen.yml

@@ -44,6 +44,24 @@ config:
     DateTime: string
     JSON: any  
 generates:
+  './packages/data-context/src/gen/all-operations.gen.ts':
+    config:
+      <<: *documentFilters
+      flattenGeneratedTypes: true
+    schema: 'packages/graphql/schemas/schema.graphql'
+    documents:
+      - './packages/frontend-shared/src/gql-components/**/*.vue'
+      - './packages/app/src/**/*.vue'
+      - './packages/launchpad/src/**/*.vue'
+    plugins:
+      - add:
+          content: '/* eslint-disable */'
+      - 'typescript':
+          noExport: true
+      - 'typescript-operations':
+          noExport: true
+      - 'typed-document-node'
+
   ###
   # Generates types for us to infer the correct "source types" when we mock out on the frontend
   # This ensures we have proper type checking when we're using cy.mountFragment in component tests

packages/data-context/package.json

@@ -14,6 +14,7 @@
   },
   "dependencies": {
     "@storybook/csf-tools": "^6.4.0-alpha.38",
+    "create-cypress-tests": "0.0.0-development",
     "cross-fetch": "^3.1.4",
     "dataloader": "^2.0.0",
     "globby": "^11.0.1",
@@ -22,9 +23,9 @@
     "wonka": "^4.0.15"
   },
   "devDependencies": {
+    "@packages/resolve-dist": "0.0.0-development",
     "@packages/ts": "0.0.0-development",
     "@packages/types": "0.0.0-development",
-    "create-cypress-tests": "0.0.0-development",
     "mocha": "7.0.1",
     "rimraf": "3.0.2"
   },

packages/data-context/src/DataContext.ts

@@ -1,10 +1,8 @@
 import type { LaunchArgs, OpenProjectLaunchOptions, PlatformName } from '@packages/types'
-import path from 'path'
 import type { AppApiShape, ProjectApiShape } from './actions'
 import type { NexusGenAbstractTypeMembers } from '@packages/graphql/src/gen/nxs.gen'
 import type { AuthApiShape } from './actions/AuthActions'
 import debugLib from 'debug'
-import fsExtra from 'fs-extra'
 import { CoreDataShape, makeCoreData } from './data/coreDataShape'
 import { DataActions } from './DataActions'
 import {
@@ -14,16 +12,17 @@ import {
   ProjectDataSource,
   WizardDataSource,
   BrowserDataSource,
-  UtilDataSource,
   StorybookDataSource,
   CloudDataSource,
 } from './sources/'
 import { cached } from './util/cached'
 import { DataContextShell, DataContextShellConfig } from './DataContextShell'
+import type { GraphQLSchema } from 'graphql'
 
 const IS_DEV_ENV = process.env.CYPRESS_INTERNAL_ENV !== 'production'
 
 export interface DataContextConfig extends DataContextShellConfig {
+  schema: GraphQLSchema
   os: PlatformName
   launchArgs: LaunchArgs
   launchOptions: OpenProjectLaunchOptions
@@ -42,16 +41,6 @@ export interface DataContextConfig extends DataContextShellConfig {
 export class DataContext extends DataContextShell {
   private _coreData: CoreDataShape
 
-  @cached
-  get fs () {
-    return fsExtra
-  }
-
-  @cached
-  get path () {
-    return path
-  }
-
   constructor (private config: DataContextConfig) {
     super(config)
     this._coreData = config.coreData ?? makeCoreData()
@@ -114,11 +103,6 @@ export class DataContext extends DataContextShell {
     return this.coreData.baseError
   }
 
-  @cached
-  get util () {
-    return new UtilDataSource(this)
-  }
-
   @cached
   get file () {
     return new FileDataSource(this)

packages/data-context/src/DataContextShell.ts

@@ -1,11 +1,18 @@
 import { EventEmitter } from 'events'
+import type { GraphQLSchema } from 'graphql'
 import type { Server } from 'http'
 import type { AddressInfo } from 'net'
+import path from 'path'
+import fsExtra from 'fs-extra'
+
 import { DataEmitterActions } from './actions/DataEmitterActions'
+import { GraphQLDataSource, HtmlDataSource, UtilDataSource } from './sources'
+import { EnvDataSource } from './sources/EnvDataSource'
 import { cached } from './util/cached'
 
 export interface DataContextShellConfig {
   rootBus: EventEmitter
+  schema: GraphQLSchema
 }
 
 // Used in places where we have to create a "shell" data context,
@@ -15,7 +22,12 @@ export class DataContextShell {
   private _appServerPort: number | undefined
   private _gqlServerPort: number | undefined
 
-  constructor (private shellConfig: DataContextShellConfig = { rootBus: new EventEmitter }) {}
+  constructor (private shellConfig: DataContextShellConfig = { rootBus: new EventEmitter, schema: require('@packages/graphql').graphqlSchema }) {}
+
+  @cached
+  get fs () {
+    return fsExtra
+  }
 
   setAppServerPort (port: number | undefined) {
     this._appServerPort = port
@@ -34,11 +46,36 @@ export class DataContextShell {
     return this._gqlServerPort
   }
 
+  @cached
+  get path () {
+    return path
+  }
+
+  @cached
+  get env () {
+    return new EnvDataSource(this)
+  }
+
   @cached
   get emitter () {
     return new DataEmitterActions(this)
   }
 
+  @cached
+  get graphql () {
+    return new GraphQLDataSource(this, this.shellConfig.schema)
+  }
+
+  @cached
+  get html () {
+    return new HtmlDataSource(this)
+  }
+
+  @cached
+  get util () {
+    return new UtilDataSource(this)
+  }
+
   get _apis () {
     return {
       busApi: this.shellConfig.rootBus,

packages/data-context/src/sources/CloudDataSource.ts

@@ -31,10 +31,10 @@ export interface CloudExecuteRemote {
 }
 
 export class CloudDataSource {
-  private _urqlClient: Client
+  private _cloudUrqlClient: Client
 
   constructor (private ctx: DataContext) {
-    this._urqlClient = createClient({
+    this._cloudUrqlClient = createClient({
       url: `${REMOTE_SCHEMA_URLS[cloudEnv]}/test-runner-graphql`,
       exchanges: [
         dedupExchange,
@@ -58,7 +58,7 @@ export class CloudDataSource {
 
     const requestPolicy = config.requestPolicy ?? 'cache-and-network'
 
-    const executingQuery = this._urqlClient.executeQuery(createRequest(config.query, config.variables), {
+    const executingQuery = this._cloudUrqlClient.executeQuery(createRequest(config.query, config.variables), {
       fetch: this.ctx.util.fetch,
       requestPolicy,
       fetchOptions: {

packages/data-context/src/sources/EnvDataSource.ts

@@ -0,0 +1,12 @@
+import type { DataContextShell } from '../DataContextShell'
+
+/**
+ * Centralizes all of the "env"
+ */
+export class EnvDataSource {
+  constructor (private ctx: DataContextShell) {}
+
+  get CYPRESS_INTERNAL_VITE_APP_PORT () {
+    return process.env.CYPRESS_INTERNAL_VITE_APP_PORT
+  }
+}

packages/data-context/src/sources/GraphQLDataSource.ts

@@ -0,0 +1,52 @@
+import { createClient, Client, dedupExchange, ssrExchange } from '@urql/core'
+import { cacheExchange } from '@urql/exchange-graphcache'
+import { executeExchange } from '@urql/exchange-execute'
+import type { GraphQLSchema } from 'graphql'
+import type { DataContextShell } from '../DataContextShell'
+import type * as allOperations from '../gen/all-operations.gen'
+
+type AllQueries<T> = {
+  [K in keyof T]: K
+}[keyof T]
+
+export class GraphQLDataSource {
+  private _urqlClient: Client
+  private _ssr: ReturnType<typeof ssrExchange>
+
+  constructor (private ctx: DataContextShell, private schema: GraphQLSchema) {
+    this._ssr = ssrExchange({ isClient: false })
+    this._urqlClient = this.makeClient()
+  }
+
+  resetClient () {
+    this._urqlClient = this.makeClient()
+  }
+
+  private _allQueries?: typeof allOperations
+
+  executeQuery (document: AllQueries<typeof allOperations>, variables: Record<string, any>) {
+    // Late require'd to avoid erroring if codegen hasn't run (for legacy Cypress workflow)
+    const allQueries = (this._allQueries ??= require('../gen/all-operations.gen'))
+
+    return this._urqlClient.query(allQueries[document], variables).toPromise()
+  }
+
+  getSSRData () {
+    return this._ssr.extractData()
+  }
+
+  private makeClient () {
+    return createClient({
+      url: `__`,
+      exchanges: [
+        dedupExchange,
+        cacheExchange(),
+        this._ssr,
+        executeExchange({
+          schema: this.schema,
+          context: this.ctx,
+        }),
+      ],
+    })
+  }
+}

packages/data-context/src/sources/HtmlDataSource.ts

@@ -0,0 +1,56 @@
+/**
+ * Forms the HTML pages rendered to the client for Component / E2E testing,
+ * including the pre-hydration we use to bootstrap the script data for fast
+ * initial loading
+ */
+import type { DataContextShell } from '../DataContextShell'
+import { getPathToDist } from '@packages/resolve-dist'
+
+export class HtmlDataSource {
+  constructor (private ctx: DataContextShell) {}
+
+  async fetchAppInitialData () {
+    await Promise.all([
+      this.ctx.graphql.executeQuery('AppQueryDocument', {}),
+      this.ctx.graphql.executeQuery('NewSpec_NewSpecQueryDocument', {}),
+      this.ctx.graphql.executeQuery('ProjectSettingsDocument', {}),
+      this.ctx.graphql.executeQuery('SpecsPageContainerDocument', {}),
+      this.ctx.graphql.executeQuery('HeaderBar_HeaderBarQueryDocument', {}),
+    ])
+
+    return this.ctx.graphql.getSSRData()
+  }
+
+  async fetchAppHtml () {
+    if (this.ctx.env.CYPRESS_INTERNAL_VITE_APP_PORT) {
+      const response = await this.ctx.util.fetch(`http://localhost:${process.env.CYPRESS_INTERNAL_VITE_APP_PORT}/`, { method: 'GET' })
+      const html = await response.text()
+
+      return html
+    }
+
+    return this.ctx.fs.readFile(getPathToDist('app'), 'utf8')
+  }
+
+  /**
+   * The app html includes the SSR'ed data to bootstrap the page for the app
+   */
+  async appHtml () {
+    const [appHtml, appInitialData] = await Promise.all([
+      this.fetchAppHtml(),
+      this.fetchAppInitialData(),
+    ])
+
+    return this.replaceBody(appHtml, appInitialData)
+  }
+
+  private replaceBody (html: string, initialData: object) {
+    return html.replace('<body>', `
+      <body>
+        <script>
+          window.__CYPRESS_GRAPHQL_PORT__ = ${JSON.stringify(this.ctx.gqlServerPort)};
+          window.__CYPRESS_INITIAL_DATA__ = ${JSON.stringify(initialData)};
+        </script>
+    `)
+  }
+}

packages/data-context/src/sources/UtilDataSource.ts

@@ -1,8 +1,7 @@
 import DataLoader from 'dataloader'
 import crypto from 'crypto'
 import fetch from 'cross-fetch'
-
-import type { DataContext } from '..'
+import type { DataContextShell } from '../DataContextShell'
 
 /**
  * this.ctx.util....
@@ -11,7 +10,7 @@ import type { DataContext } from '..'
  * within the DataContext layer
  */
 export class UtilDataSource {
-  constructor (private ctx: DataContext) {}
+  constructor (private ctx: DataContextShell) {}
 
   private _allLoaders: DataLoader<any, any>[] = []
 

packages/data-context/src/sources/index.ts

@@ -4,8 +4,11 @@
 export * from './AppDataSource'
 export * from './BrowserDataSource'
 export * from './CloudDataSource'
+export * from './EnvDataSource'
 export * from './FileDataSource'
 export * from './GitDataSource'
+export * from './GraphQLDataSource'
+export * from './HtmlDataSource'
 export * from './ProjectDataSource'
 export * from './SettingsDataSource'
 export * from './StorybookDataSource'

packages/frontend-shared/cypress/e2e/support/e2eSupport.ts

@@ -127,7 +127,17 @@ function visitApp (href?: string) {
     throw new Error(`Missing serverPort - did you forget to call cy.initializeApp(...) ?`)
   }
 
-  return cy.visit(`dist-app/index.html?gqlPort=${e2e_gqlPort}&serverPort=${e2e_serverPort}${href || ''}`)
+  cy.withCtx(async (ctx) => {
+    return JSON.stringify(ctx.html.fetchAppInitialData())
+  }, { log: false }).then((ssrData) => {
+    return cy.visit(`dist-app/index.html?serverPort=${e2e_serverPort}${href || ''}`, {
+      onBeforeLoad (win) {
+        // Simulates the inject SSR data when we're loading the page normally in the app
+        win.__CYPRESS_INITIAL_DATA__ = JSON.parse(ssrData)
+        win.__CYPRESS_GRAPHQL_PORT__ = e2e_gqlPort
+      },
+    })
+  })
 }
 
 function visitLaunchpad (hash?: string) {
@@ -142,7 +152,9 @@ function visitLaunchpad (hash?: string) {
 
 const pageLoadId = `uid${Math.random()}`
 
-function withCtx<T extends Partial<WithCtxOptions>> (fn: (ctx: DataContext, o: T & WithCtxInjected) => any, opts: T = {} as T): Cypress.Chainable {
+type UnwrapPromise<R> = R extends PromiseLike<infer U> ? U : R
+
+function withCtx<T extends Partial<WithCtxOptions>, R> (fn: (ctx: DataContext, o: T & WithCtxInjected) => R, opts: T = {} as T): Cypress.Chainable<UnwrapPromise<R>> {
   const _log = opts.log === false ? { end () {} } : Cypress.log({
     name: 'withCtx',
     message: '(view in console)',
@@ -155,13 +167,15 @@ function withCtx<T extends Partial<WithCtxOptions>> (fn: (ctx: DataContext, o: T
 
   const { log, timeout, ...rest } = opts
 
-  return cy.task('withCtx', {
+  return cy.task<UnwrapPromise<R>>('withCtx', {
     fn: fn.toString(),
     options: rest,
     // @ts-expect-error
     activeTestId: `${pageLoadId}-${Cypress.mocha.getRunner().test.id ?? Cypress.currentTest.title}`,
-  }, { timeout: timeout ?? Cypress.env('e2e_isDebugging') ? NO_TIMEOUT : FOUR_SECONDS, log }).then(() => {
+  }, { timeout: timeout ?? Cypress.env('e2e_isDebugging') ? NO_TIMEOUT : FOUR_SECONDS, log }).then((result) => {
     _log.end()
+
+    return result
   })
 }
 

packages/frontend-shared/src/graphql/urqlClient.ts

@@ -5,7 +5,9 @@ import {
   errorExchange,
   fetchExchange,
   Exchange,
+  ssrExchange,
 } from '@urql/core'
+import type { SSRData } from '@urql/core/dist/types/exchanges/ssr'
 import { devtoolsExchange } from '@urql/devtools'
 import { useToast } from 'vue-toastification'
 import { client } from '@packages/socket/lib/browser'
@@ -32,17 +34,21 @@ export function makeCacheExchange () {
   })
 }
 
+declare global {
+  interface Window {
+    __CYPRESS_INITIAL_DATA__: SSRData
+    __CYPRESS_GRAPHQL_PORT__?: string
+  }
+}
+
 export function makeUrqlClient (target: 'launchpad' | 'app'): Client {
   let gqlPort: string
 
   if (GQL_PORT_MATCH) {
     gqlPort = GQL_PORT_MATCH[1]
-  } else {
-    // @ts-ignore
+  } else if (window.__CYPRESS_GRAPHQL_PORT__) {
     gqlPort = window.__CYPRESS_GRAPHQL_PORT__
-  }
-
-  if (!gqlPort) {
+  } else {
     throw new Error(`${window.location.href} cannot be visited without a gqlPort`)
   }
 
@@ -77,6 +83,10 @@ export function makeUrqlClient (target: 'launchpad' | 'app'): Client {
     // https://formidable.com/open-source/urql/docs/graphcache/errors/
     makeCacheExchange(),
     namedRouteExchange,
+    ssrExchange({
+      isClient: true,
+      initialState: window.__CYPRESS_INITIAL_DATA__ ?? {},
+    }),
     // TODO(tim): add this when we want to use the socket as the GraphQL
     // transport layer for all operations
     // target === 'launchpad' ? fetchExchange : socketExchange(io),

packages/server/lib/makeDataContext.ts

@@ -8,6 +8,7 @@ import * as config from './config'
 import type { EventEmitter } from 'events'
 import { openProject } from './open_project'
 import cache from './cache'
+import { graphqlSchema } from '@packages/graphql/src/schema'
 
 const { getBrowsers } = browserUtils
 
@@ -19,6 +20,7 @@ interface MakeDataContextOptions {
 
 export function makeDataContext (options: MakeDataContextOptions) {
   return new DataContext({
+    schema: graphqlSchema,
     ...options,
     launchOptions: {},
     appApi: {

packages/server/lib/routes.ts

@@ -1,7 +1,7 @@
 import httpProxy from 'http-proxy'
 import _ from 'lodash'
 import Debug from 'debug'
-import { ErrorRequestHandler, Router, Response } from 'express'
+import { ErrorRequestHandler, Router } from 'express'
 import send from 'send'
 import { getPathToDist } from '@packages/resolve-dist'
 
@@ -94,39 +94,15 @@ export const createCommonRoutes = ({
       target: `http://localhost:${process.env.CYPRESS_INTERNAL_VITE_APP_PORT}/`,
     })
 
-    const proxyIndex = httpProxy.createProxyServer({
-      target: `http://localhost:${process.env.CYPRESS_INTERNAL_VITE_APP_PORT}/`,
-      selfHandleResponse: true,
-    })
-
-    proxyIndex.on('proxyRes', function (proxyRes, _req, _res) {
-      const body: any[] = []
-
-      proxyRes.on('data', function (chunk) {
-        let chunkData = String(chunk)
-
-        if (chunkData.includes('<head>')) {
-          chunkData = chunkData.replace('<body>', replaceBody(ctx))
-        }
-
-        body.push(chunkData)
-      })
-
-      proxyRes.on('end', function () {
-        (_res as Response).send(body.join(''))
-      })
+    router.get('/__vite__/', (req, res) => {
+      ctx.html.appHtml().then((html) => res.send(html)).catch((e) => res.status(500).send({ stack: e.stack }))
     })
 
     // TODO: can namespace this onto a "unified" route like __app-unified__
     // make sure to update the generated routes inside of vite.config.ts
     router.get('/__vite__/*', (req, res) => {
       debug('Proxy to __vite__')
-
-      if (req.params[0] === '') {
-        proxyIndex.web(req, res, {}, (e) => {})
-      } else {
-        proxy.web(req, res, {}, (e) => {})
-      }
+      proxy.web(req, res, {}, (e) => {})
     })
   } else {
     router.get('/__vite__/*', (req, res) => {

packages/server/lib/server-e2e.ts

@@ -3,7 +3,6 @@ import Debug from 'debug'
 import isHtml from 'is-html'
 import _ from 'lodash'
 import stream from 'stream'
-import { EventEmitter } from 'events'
 import url from 'url'
 import httpsProxy from '@packages/https-proxy'
 import { getRouteForRequest } from '@packages/net-stubbing'
@@ -46,7 +45,7 @@ const isResponseHtml = function (contentType, responseBuffer) {
 export class ServerE2E extends ServerBase<SocketE2E> {
   private _urlResolver: Bluebird<Record<string, any>> | null
 
-  constructor (ctx: DataContextShell = new DataContextShell({ rootBus: new EventEmitter })) {
+  constructor (ctx: DataContextShell = new DataContextShell()) {
     super(ctx)
 
     this._urlResolver = null

scripts/binary/util/packages.js

@@ -168,12 +168,12 @@ const replaceLocalNpmVersions = function (basePath) {
 
         if (dependencies) {
           return Promise.all(_.map(dependencies, (version, pkg) => {
-            const parsedPkg = /(@cypress\/)(.*)/g.exec(pkg)
+            const matchedPkg = Boolean(pkg.startsWith('@cypress/') || pkg === 'create-cypress-tests')
 
-            if (parsedPkg && parsedPkg.length === 3 && version === '0.0.0-development') {
-              const pkgName = parsedPkg[2]
+            if (matchedPkg && version === '0.0.0-development') {
+              const pkgName = pkg.startsWith('@cypress/') ? pkg.split('/')[1] : pkg
 
-              json.dependencies[`@cypress/${pkgName}`] = `file:${path.join(basePath, 'npm', pkgName)}`
+              json.dependencies[pkg] = `file:${path.join(basePath, 'npm', pkgName)}`
               shouldWriteFile = true
 
               return updateNpmPackage(pkgName)

system-tests/lib/fixtures.js

@@ -32,7 +32,6 @@ module.exports = {
     const from = path.join(projects, project)
     const to = path.join(tmpDir, project)
 
-    console.log({ from, to })
     fs.copySync(from, to)
   },