chore: break up circle CI tasks (#1983)

.circleci/config.yml

@@ -47,6 +47,9 @@ jobs:
       - run:
           name: Unit tests
           command: yarn test-unit
+      - run:
+          name: Install mastodon
+          command: yarn install-mastodon
       - run:
           name: Wait for postgres to be ready
           command: |
@@ -67,11 +70,12 @@ jobs:
               sleep 1
             done
             echo Failed waiting for redis && exit 1
+      - run:
+          name: Build
+          command: yarn build
       - run:
           name: Integration tests
-          command: |
+          command: yarn test-in-ci
-            node -v
-            yarn test
       - save_cache:
           name: Save yarn cache
           key: yarn-v1-{{ checksum "yarn.lock" }}

bin/install-mastodon.js

@@ -0,0 +1,77 @@
+import { promisify } from 'util'
+import childProcessPromise from 'child-process-promise'
+import path from 'path'
+import fs from 'fs'
+import { DB_NAME, DB_PASS, DB_USER, mastodonDir, env } from './mastodon-config'
+import mkdirp from 'mkdirp'
+
+const exec = childProcessPromise.exec
+const stat = promisify(fs.stat)
+const writeFile = promisify(fs.writeFile)
+const dir = __dirname
+
+async function setupMastodonDatabase () {
+  console.log('Setting up mastodon database...')
+  try {
+    await exec(`psql -d template1 -c "CREATE USER ${DB_USER} WITH PASSWORD '${DB_PASS}' CREATEDB;"`)
+  } catch (e) { /* ignore */ }
+  try {
+    await exec(`dropdb -h 127.0.0.1 -U ${DB_USER} -w ${DB_NAME}`, {
+      cwd: mastodonDir,
+      env: Object.assign({ PGPASSWORD: DB_PASS }, process.env)
+    })
+  } catch (e) { /* ignore */ }
+  await exec(`createdb -h 127.0.0.1 -U ${DB_USER} -w ${DB_NAME}`, {
+    cwd: mastodonDir,
+    env: Object.assign({ PGPASSWORD: DB_PASS }, process.env)
+  })
+
+  const dumpFile = path.join(dir, '../tests/fixtures/dump.sql')
+  await exec(`psql -h 127.0.0.1 -U ${DB_USER} -w -d ${DB_NAME} -f "${dumpFile}"`, {
+    cwd: mastodonDir,
+    env: Object.assign({ PGPASSWORD: DB_PASS }, process.env)
+  })
+
+  const tgzFile = path.join(dir, '../tests/fixtures/system.tgz')
+  const systemDir = path.join(mastodonDir, 'public/system')
+  await mkdirp(systemDir)
+  await exec(`tar -xzf "${tgzFile}"`, { cwd: systemDir })
+}
+
+async function installMastodonDependencies () {
+  const cwd = mastodonDir
+  const installCommands = [
+    'gem update --system',
+    'gem install bundler foreman',
+    'bundle config set --local frozen \'true\'',
+    'bundle install',
+    'yarn --pure-lockfile'
+  ]
+
+  const installedFile = path.join(mastodonDir, 'installed.txt')
+  try {
+    await stat(installedFile)
+    console.log('Already installed Mastodon dependencies')
+  } catch (e) {
+    console.log('Installing Mastodon dependencies...')
+    for (const cmd of installCommands) {
+      console.log(cmd)
+      await exec(cmd, { cwd, env })
+    }
+    await writeFile(installedFile, '', 'utf8')
+  }
+  await exec('bundle exec rails db:migrate', { cwd, env })
+}
+
+export default async function installMastodon () {
+  console.log('Installing Mastodon...')
+  await setupMastodonDatabase()
+  await installMastodonDependencies()
+}
+
+if (require.main === module) {
+  installMastodon().catch(err => {
+    console.error(err)
+    process.exit(1)
+  })
+}

bin/mastodon-config.js

@@ -1,3 +1,5 @@
+import path from 'path'
+
 export const DB_NAME = 'pinafore_development'
 export const DB_USER = 'pinafore'
 export const DB_PASS = 'pinafore'
@@ -18,3 +20,16 @@ DB_PASS=${DB_PASS}
 // Need a Ruby version that CircleCI bundles with Node v12, not Node v14 which doesn't
 // work for streaming
 export const RUBY_VERSION = '2.6.6'
+
+export const mastodonDir = path.join(__dirname, '../mastodon')
+
+export const env = Object.assign({}, process.env, {
+  RAILS_ENV: 'development',
+  NODE_ENV: 'development',
+  BUNDLE_PATH: path.join(mastodonDir, 'vendor/bundle'),
+  DB_NAME,
+  DB_USER,
+  DB_PASS,
+  DB_HOST,
+  DB_PORT
+})

bin/run-mastodon.js

@@ -1,84 +1,18 @@
 import { restoreMastodonData } from './restore-mastodon-data'
-import { promisify } from 'util'
 import childProcessPromise from 'child-process-promise'
-import path from 'path'
 import fs from 'fs'
 import { waitForMastodonUiToStart, waitForMastodonApiToStart } from './wait-for-mastodon-to-start'
-import mkdirp from 'mkdirp'
 import cloneMastodon from './clone-mastodon'
-import { DB_USER, DB_PASS, DB_NAME, DB_HOST, DB_PORT } from './mastodon-config'
+import installMastodon from './install-mastodon'
+import { mastodonDir, env } from './mastodon-config'
 
-const exec = childProcessPromise.exec
 const spawn = childProcessPromise.spawn
-const stat = promisify(fs.stat)
-const writeFile = promisify(fs.writeFile)
-const dir = __dirname
-const mastodonDir = path.join(dir, '../mastodon')
 
 let childProc
 
-async function setupMastodonDatabase () {
-  console.log('Setting up mastodon database...')
-  try {
-    await exec(`psql -d template1 -c "CREATE USER ${DB_USER} WITH PASSWORD '${DB_PASS}' CREATEDB;"`)
-  } catch (e) { /* ignore */ }
-  try {
-    await exec(`dropdb -h 127.0.0.1 -U ${DB_USER} -w ${DB_NAME}`, {
-      cwd: mastodonDir,
-      env: Object.assign({ PGPASSWORD: DB_PASS }, process.env)
-    })
-  } catch (e) { /* ignore */ }
-  await exec(`createdb -h 127.0.0.1 -U ${DB_USER} -w ${DB_NAME}`, {
-    cwd: mastodonDir,
-    env: Object.assign({ PGPASSWORD: DB_PASS }, process.env)
-  })
-
-  const dumpFile = path.join(dir, '../tests/fixtures/dump.sql')
-  await exec(`psql -h 127.0.0.1 -U ${DB_USER} -w -d ${DB_NAME} -f "${dumpFile}"`, {
-    cwd: mastodonDir,
-    env: Object.assign({ PGPASSWORD: DB_PASS }, process.env)
-  })
-
-  const tgzFile = path.join(dir, '../tests/fixtures/system.tgz')
-  const systemDir = path.join(mastodonDir, 'public/system')
-  await mkdirp(systemDir)
-  await exec(`tar -xzf "${tgzFile}"`, { cwd: systemDir })
-}
-
 async function runMastodon () {
   console.log('Running mastodon...')
-  const env = Object.assign({}, process.env, {
-    RAILS_ENV: 'development',
-    NODE_ENV: 'development',
-    BUNDLE_PATH: path.join(mastodonDir, 'vendor/bundle'),
-    DB_NAME,
-    DB_USER,
-    DB_PASS,
-    DB_HOST,
-    DB_PORT
-  })
   const cwd = mastodonDir
-  const installCommands = [
-    'gem update --system',
-    'gem install bundler foreman',
-    'bundle config set --local frozen \'true\'',
-    'bundle install',
-    'yarn --pure-lockfile'
-  ]
-
-  const installedFile = path.join(mastodonDir, 'installed.txt')
-  try {
-    await stat(installedFile)
-    console.log('Already installed Mastodon')
-  } catch (e) {
-    console.log('Installing Mastodon...')
-    for (const cmd of installCommands) {
-      console.log(cmd)
-      await exec(cmd, { cwd, env })
-    }
-    await writeFile(installedFile, '', 'utf8')
-  }
-  await exec('bundle exec rails db:migrate', { cwd, env })
   const promise = spawn('foreman', ['start'], { cwd, env })
   // don't bother writing to mastodon.log in CI; we can't read the file anyway
   const logFile = process.env.CIRCLECI ? '/dev/null' : 'mastodon.log'
@@ -96,7 +30,7 @@ async function runMastodon () {
 
 async function main () {
   await cloneMastodon()
-  await setupMastodonDatabase()
+  await installMastodon()
   await runMastodon()
   await waitForMastodonApiToStart()
   await restoreMastodonData()

package.json

@@ -18,6 +18,7 @@
     "build-template-html-watch": "node -r esm ./bin/build-template-html.js --watch",
     "build-assets": "node -r esm ./bin/build-assets.js",
     "clone-mastodon": "node -r esm ./bin/clone-mastodon.js",
+    "install-mastodon": "node -r esm ./bin/install-mastodon.js",
     "run-mastodon": "node -r esm ./bin/run-mastodon.js",
     "test": "cross-env BROWSER=chrome:headless run-s test-browser",
     "test-browser": "run-p --race run-mastodon build-and-start test-mastodon",
@@ -26,6 +27,7 @@
     "testcafe-suite0": "cross-env-shell testcafe -c 4 $BROWSER tests/spec/0*",
     "testcafe-suite1": "cross-env-shell testcafe $BROWSER tests/spec/1*",
     "test-unit": "NODE_ENV=test mocha -r esm -r bin/browser-shim.js tests/unit/",
+    "test-in-ci": "cross-env BROWSER=chrome:headless run-p --race run-mastodon start test-mastodon",
     "wait-for-mastodon-to-start": "node -r esm bin/wait-for-mastodon-to-start.js",
     "wait-for-mastodon-data": "node -r esm bin/wait-for-mastodon-data.js",
     "backup-mastodon-data": "./bin/backup-mastodon-data.sh",