From f3109ad8a7ddf7a162e7c9a050ffe6c3efb75491 Mon Sep 17 00:00:00 2001 From: Andrew Clark Date: Wed, 22 May 2019 11:14:14 -0700 Subject: [PATCH] Parallelize CircleCI jobs using workflows (#15704) Updates the CircleCI config to use the workflows features to run jobs in parallel, instead of the `parallelism` option. This change alone doesn't improve the overall build time much, since almost all of the total time is spent running the Rollup script, which runs entirely sequentially. But it does improve reporting, and should make it easier to add additional parallel jobs in the future. --- .circleci/config.yml | 192 +++++++++++++++++++++++---- scripts/circleci/test_entry_point.sh | 73 ---------- 2 files changed, 164 insertions(+), 101 deletions(-) delete mode 100755 scripts/circleci/test_entry_point.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 6c0b662c1f..6d45ac5a39 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,53 +1,189 @@ version: 2 + +aliases: + - &docker + - image: circleci/openjdk:8-jdk-node-browsers + + - &environment + TZ: /usr/share/zoneinfo/America/Los_Angeles + + - &restore_node_modules + name: Restore node_modules cache + keys: + - v1-node-{{ arch }}-{{ .Branch }}-{{ checksum "yarn.lock" }} + - v1-node-{{ arch }}-{{ .Branch }}- + - v1-node-{{ arch }}- + - &attach_workspace + at: build + jobs: - build: - - docker: - - image: circleci/openjdk:8-jdk-node-browsers - - environment: - TZ: /usr/share/zoneinfo/America/Los_Angeles - - parallelism: 4 + setup: + docker: *docker + environment: *environment steps: - checkout - - - run: echo $CIRCLE_COMPARE_URL | cut -d/ -f7 - - - restore_cache: - name: Restore node_modules cache - keys: - - v1-node-{{ arch }}-{{ .Branch }}-{{ checksum "yarn.lock" }} - - v1-node-{{ arch }}-{{ .Branch }}- - - v1-node-{{ arch }}- - + - restore_cache: *restore_node_modules - run: name: Nodejs Version command: node --version - - run: name: Install Packages command: yarn install --frozen-lockfile - - - run: - name: Test Packages - command: ./scripts/circleci/test_entry_point.sh - - save_cache: name: Save node_modules cache key: v1-node-{{ arch }}-{{ .Branch }}-{{ checksum "yarn.lock" }} paths: - node_modules + lint: + docker: *docker + environment: *environment + + steps: + - checkout + - restore_cache: *restore_node_modules + - run: node ./scripts/prettier/index + - run: node ./scripts/tasks/eslint + - run: ./scripts/circleci/check_license.sh + - run: ./scripts/circleci/check_modules.sh + - run: ./scripts/circleci/test_print_warnings.sh + + flow: + docker: *docker + environment: *environment + + steps: + - checkout + - restore_cache: *restore_node_modules + - run: node ./scripts/tasks/flow-ci + + test-source: + docker: *docker + environment: *environment + + steps: + - checkout + - restore_cache: *restore_node_modules + - run: yarn test --maxWorkers=2 + + test-source-persistent: + docker: *docker + environment: *environment + + steps: + - checkout + - restore_cache: *restore_node_modules + - run: yarn test-persistent --maxWorkers=2 + + test-source-prod: + docker: *docker + environment: *environment + + steps: + - checkout + - restore_cache: *restore_node_modules + - run: yarn test-prod --maxWorkers=2 + + test-source-fire: + docker: *docker + environment: *environment + + steps: + - checkout + - restore_cache: *restore_node_modules + - run: yarn test-fire --maxWorkers=2 + - run: yarn test-fire-prod --maxWorkers=2 + + test-coverage: + docker: *docker + environment: *environment + + steps: + - checkout + - restore_cache: *restore_node_modules + - run: ./scripts/circleci/test_coverage.sh + + build: + docker: *docker + environment: *environment + steps: + - checkout + - restore_cache: *restore_node_modules + - run: ./scripts/circleci/add_build_info_json.sh + - run: ./scripts/circleci/update_package_versions.sh + - run: ./scripts/circleci/build.sh + - run: yarn test-build --maxWorkers=2 + - run: yarn test-build-prod --maxWorkers=2 + - run: cp ./scripts/rollup/results.json ./build/bundle-sizes.json + - run: node ./scripts/tasks/danger + - run: ./scripts/circleci/upload_build.sh + - run: ./scripts/circleci/pack_and_store_artifact.sh - store_artifacts: path: ./node_modules.tgz - - store_artifacts: path: ./build.tgz - - store_artifacts: path: ./build/bundle-sizes.json - - store_artifacts: path: ./scripts/error-codes/codes.json + - persist_to_workspace: + root: build + paths: + - facebook-www + - node_modules + - react-native + + test-build: + docker: *docker + environment: *environment + steps: + - checkout + - attach_workspace: *attach_workspace + - restore_cache: *restore_node_modules + - run: yarn test-build --maxWorkers=2 + + test-build-prod: + docker: *docker + environment: *environment + steps: + - checkout + - attach_workspace: *attach_workspace + - restore_cache: *restore_node_modules + - run: yarn test-build-prod --maxWorkers=2 + +workflows: + version: 2 + build-and-test: + jobs: + - setup + - lint: + requires: + - setup + - flow: + requires: + - setup + - test-source: + requires: + - setup + - test-source-prod: + requires: + - setup + - test-source-persistent: + requires: + - setup + - test-source-fire: + requires: + - setup + - test-coverage: + requires: + - setup + - build: + requires: + - setup + - test-build: + requires: + - build + - test-build-prod: + requires: + - build diff --git a/scripts/circleci/test_entry_point.sh b/scripts/circleci/test_entry_point.sh deleted file mode 100755 index 5bf568028a..0000000000 --- a/scripts/circleci/test_entry_point.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/bash - -set -e - -./scripts/circleci/set_up_github_keys.sh - -COMMANDS_TO_RUN=() - -if [ $((0 % CIRCLE_NODE_TOTAL)) -eq "$CIRCLE_NODE_INDEX" ]; then - COMMANDS_TO_RUN+=('node ./scripts/prettier/index') - COMMANDS_TO_RUN+=('node ./scripts/tasks/flow-ci') - COMMANDS_TO_RUN+=('node ./scripts/tasks/eslint') - COMMANDS_TO_RUN+=('yarn test --maxWorkers=2') - COMMANDS_TO_RUN+=('yarn test-persistent --maxWorkers=2') - COMMANDS_TO_RUN+=('./scripts/circleci/check_license.sh') - COMMANDS_TO_RUN+=('./scripts/circleci/check_modules.sh') - COMMANDS_TO_RUN+=('./scripts/circleci/test_print_warnings.sh') -fi - -if [ $((1 % CIRCLE_NODE_TOTAL)) -eq "$CIRCLE_NODE_INDEX" ]; then - COMMANDS_TO_RUN+=('yarn test-prod --maxWorkers=2') - # React Fire: - COMMANDS_TO_RUN+=('yarn test-fire --maxWorkers=2') - COMMANDS_TO_RUN+=('yarn test-fire-prod --maxWorkers=2') -fi - -if [ $((2 % CIRCLE_NODE_TOTAL)) -eq "$CIRCLE_NODE_INDEX" ]; then - COMMANDS_TO_RUN+=('./scripts/circleci/add_build_info_json.sh') - COMMANDS_TO_RUN+=('./scripts/circleci/update_package_versions.sh') - COMMANDS_TO_RUN+=('./scripts/circleci/build.sh') - COMMANDS_TO_RUN+=('yarn test-build --maxWorkers=2') - COMMANDS_TO_RUN+=('yarn test-build-prod --maxWorkers=2') - COMMANDS_TO_RUN+=('cp ./scripts/rollup/results.json ./build/bundle-sizes.json') - COMMANDS_TO_RUN+=('node ./scripts/tasks/danger') - COMMANDS_TO_RUN+=('./scripts/circleci/upload_build.sh') - COMMANDS_TO_RUN+=('./scripts/circleci/pack_and_store_artifact.sh') -fi - -if [ $((3 % CIRCLE_NODE_TOTAL)) -eq "$CIRCLE_NODE_INDEX" ]; then - COMMANDS_TO_RUN+=('./scripts/circleci/test_coverage.sh') -fi - -RETURN_CODES=() -FAILURE=0 - -printf "Node #%s (%s total). " "$CIRCLE_NODE_INDEX" "$CIRCLE_NODE_TOTAL" -if [ -n "${COMMANDS_TO_RUN[0]}" ]; then - echo "Preparing to run commands:" - for cmd in "${COMMANDS_TO_RUN[@]}"; do - echo "- $cmd" - done - - for cmd in "${COMMANDS_TO_RUN[@]}"; do - echo - echo "$ $cmd" - set +e - $cmd - rc=$? - set -e - RETURN_CODES+=($rc) - if [ $rc -ne 0 ]; then - FAILURE=$rc - fi - done - - echo - for i in "${!COMMANDS_TO_RUN[@]}"; do - echo "Received return code ${RETURN_CODES[i]} from: ${COMMANDS_TO_RUN[i]}" - done - exit $FAILURE -else - echo "No commands to run." -fi