Files
react/.github/workflows/commit_artifacts.yml
Ricky 6b35ff766a Move @generated from build to sync (#29799)
## Overview

Reverts https://github.com/facebook/react/pull/26616 and implements the
suggested way instead.

This change in #26616 broken the internal sync command, which now
results in duplicated `@generated` headers. It also makes it harder to
detect changes during the diff train sync. Instead, we will check for
changes, and if there are changes sign the files and commit them to the
sync branch.

## Strategy

The new sync strategy accounts for the generated headers during the
sync:
- **Revert Version**: Revert the version strings
- **Revert @generated**: Re-sign the files (will be the same hash as
before if unchanged)
- **Check**: Check if there are changes **if not, skip**
- **Re-apply Version**: Now add back the new version string
- **Re-sign @generated**: And re-generate the headers

Then commit to branch. This ensures that if there are no changes, we'll
skip.

---------

Co-authored-by: Timothy Yung <yungsters@gmail.com>
2024-06-10 10:16:40 -04:00

547 lines
25 KiB
YAML

name: Commit Artifacts for Meta WWW and fbsource
on:
push:
branches: [main, meta-www, meta-fbsource]
jobs:
download_artifacts:
runs-on: ubuntu-latest
outputs:
www_branch_count: ${{ steps.check_branches.outputs.www_branch_count }}
fbsource_branch_count: ${{ steps.check_branches.outputs.fbsource_branch_count }}
last_version_classic: ${{ steps.get_last_version_www.outputs.last_version_classic }}
last_version_modern: ${{ steps.get_last_version_www.outputs.last_version_modern }}
last_version_rn: ${{ steps.get_last_version_rn.outputs.last_version_rn }}
current_version_classic: ${{ steps.get_current_version.outputs.current_version_classic }}
current_version_modern: ${{ steps.get_current_version.outputs.current_version_modern }}
current_version_rn: ${{ steps.get_current_version.outputs.current_version_rn }}
steps:
- uses: actions/checkout@v4
with:
ref: builds/facebook-www
- name: "Get last version string for www"
id: get_last_version_www
run: |
# Empty checks only needed for backwards compatibility,can remove later.
VERSION_CLASSIC=$( [ -f ./compiled/facebook-www/VERSION_CLASSIC ] && cat ./compiled/facebook-www/VERSION_CLASSIC || echo '' )
VERSION_MODERN=$( [ -f ./compiled/facebook-www/VERSION_MODERN ] && cat ./compiled/facebook-www/VERSION_MODERN || echo '' )
echo "Last classic version is $VERSION_CLASSIC"
echo "Last modern version is $VERSION_MODERN"
echo "last_version_classic=$VERSION_CLASSIC" >> "$GITHUB_OUTPUT"
echo "last_version_modern=$VERSION_MODERN" >> "$GITHUB_OUTPUT"
- uses: actions/checkout@v4
with:
ref: builds/facebook-fbsource
- name: "Get last version string for rn"
id: get_last_version_rn
run: |
# Empty checks only needed for backwards compatibility,can remove later.
VERSION_NATIVE_FB=$( [ -f ./compiled-rn/VERSION_NATIVE_FB ] && cat ./compiled-rn/VERSION_NATIVE_FB || echo '' )
echo "Last rn version is $VERSION_NATIVE_FB"
echo "last_version_rn=$VERSION_NATIVE_FB" >> "$GITHUB_OUTPUT"
- uses: actions/checkout@v4
- name: "Check branches"
id: check_branches
run: |
echo "www_branch_count=$(git ls-remote --heads origin "refs/heads/meta-www" | wc -l)" >> "$GITHUB_OUTPUT"
echo "fbsource_branch_count=$(git ls-remote --heads origin "refs/heads/meta-fbsource" | wc -l)" >> "$GITHUB_OUTPUT"
- name: Download and unzip artifacts
uses: actions/github-script@v6
env:
CIRCLECI_TOKEN: ${{secrets.CIRCLECI_TOKEN_DIFFTRAIN}}
with:
script: |
// TODO: Move this to a script file.
const cp = require('child_process');
function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
function execHelper(command, options, streamStdout = false) {
return new Promise((resolve, reject) => {
const proc = cp.exec(
command,
options,
(error, stdout) => (error ? reject(error) : resolve(stdout.trim())),
);
if (streamStdout) {
proc.stdout.pipe(process.stdout);
}
});
}
let artifactsUrl = null;
// This is a temporary, dirty hack to avoid needing a GitHub auth token in the circleci
// workflow to notify this GitHub action. Sorry!
let iter = 0;
spinloop: while (iter < 15) {
const res = await github.rest.repos.listCommitStatusesForRef({
owner: context.repo.owner,
repo: context.repo.repo,
ref: context.sha
});
for (const status of res.data) {
if (/process_artifacts_combined/.test(status.context)) {
switch (status.state) {
case 'pending': {
console.log(`${status.context} is still pending`);
break;
}
case 'failure':
case 'error': {
throw new Error(`${status.context} has failed or errored`);
}
case 'success': {
// The status does not include a build ID, but we can extract it
// from the URL. I couldn't find a better way to do this.
const ciBuildId = /\/facebook\/react\/([0-9]+)/.exec(
status.target_url,
)[1];
if (Number.parseInt(ciBuildId, 10) + '' === ciBuildId) {
artifactsUrl =
`https://circleci.com/api/v1.1/project/github/facebook/react/${ciBuildId}/artifacts`;
console.log(`Found artifactsUrl: ${artifactsUrl}`);
break spinloop;
} else {
throw new Error(`${ciBuildId} isn't a number`);
}
break;
}
default: {
throw new Error(`Unhandled status state: ${status.state}`);
break;
}
}
}
}
iter++;
console.log("Sleeping for 60s...");
await sleep(60_000);
}
if (artifactsUrl != null) {
const {CIRCLECI_TOKEN} = process.env;
const res = await fetch(artifactsUrl, {
headers: {
'Circle-Token': CIRCLECI_TOKEN
}
});
const data = await res.json();
if (!Array.isArray(data) && data.message != null) {
throw `CircleCI returned: ${data.message}`;
}
for (const artifact of data) {
if (artifact.path === 'build.tgz') {
console.log(`Downloading and unzipping ${artifact.url}`);
await execHelper(
`curl -L ${artifact.url} -H "Circle-Token: ${CIRCLECI_TOKEN}" | tar -xvz`
);
}
}
} else {
process.exitCode = 1;
}
- name: Strip @license from eslint plugin and react-refresh
run: |
sed -i -e 's/ @license React*//' \
build/oss-experimental/eslint-plugin-react-hooks/cjs/eslint-plugin-react-hooks.development.js \
build/oss-experimental/react-refresh/cjs/react-refresh-babel.development.js
- name: Move relevant files for React in www into compiled
run: |
# Move the facebook-www folder into compiled
mkdir ./compiled
mv build/facebook-www ./compiled
# Move ReactAllWarnings.js to facebook-www
mkdir ./compiled/facebook-www/__test_utils__
mv build/__test_utils__/ReactAllWarnings.js ./compiled/facebook-www/__test_utils__/ReactAllWarnings.js
# Move eslint-plugin-react-hooks into facebook-www
mv build/oss-experimental/eslint-plugin-react-hooks/cjs/eslint-plugin-react-hooks.development.js \
./compiled/facebook-www/eslint-plugin-react-hooks.js
# Move unstable_server-external-runtime.js into facebook-www
mv build/oss-experimental/react-dom/unstable_server-external-runtime.js \
./compiled/facebook-www/unstable_server-external-runtime.js
# Move react-refresh-babel.development.js into babel-plugin-react-refresh
mkdir ./compiled/babel-plugin-react-refresh
mv build/oss-experimental/react-refresh/cjs/react-refresh-babel.development.js \
./compiled/babel-plugin-react-refresh/index.js
ls -R ./compiled
- name: Move relevant files for React in fbsource into compiled-rn
run: |
BASE_FOLDER='compiled-rn/facebook-fbsource/xplat/js'
mkdir -p ${BASE_FOLDER}/react-native-github/Libraries/Renderer/
mkdir -p ${BASE_FOLDER}/RKJSModules/vendor/react/{scheduler,react,react-is,react-test-renderer}/
# Move React Native renderer
mv build/react-native/implementations/ $BASE_FOLDER/react-native-github/Libraries/Renderer/
mv build/react-native/shims/ $BASE_FOLDER/react-native-github/Libraries/Renderer/
mv build/facebook-react-native/scheduler/cjs/ $BASE_FOLDER/RKJSModules/vendor/react/scheduler/
mv build/facebook-react-native/react/cjs/ $BASE_FOLDER/RKJSModules/vendor/react/react/
mv build/facebook-react-native/react-is/cjs/ $BASE_FOLDER/RKJSModules/vendor/react/react-is/
mv build/facebook-react-native/react-test-renderer/cjs/ $BASE_FOLDER/RKJSModules/vendor/react/react-test-renderer/
# Delete OSS renderer. OSS renderer is synced through internal script.
RENDERER_FOLDER=$BASE_FOLDER/react-native-github/Libraries/Renderer/implementations/
rm $RENDERER_FOLDER/ReactFabric-{dev,prod,profiling}.js
rm $RENDERER_FOLDER/ReactNativeRenderer-{dev,prod,profiling}.js
# Move React Native version file
mv build/facebook-react-native/VERSION_NATIVE_FB ./compiled-rn/VERSION_NATIVE_FB
ls -R ./compiled-rn
- name: Add REVISION files
run: |
echo ${{ github.sha }} >> ./compiled/facebook-www/REVISION
cp ./compiled/facebook-www/REVISION ./compiled/facebook-www/REVISION_TRANSFORMS
echo ${{ github.sha }} >> ./compiled-rn/facebook-fbsource/xplat/js/react-native-github/Libraries/Renderer/REVISION
- name: "Get current version string"
id: get_current_version
run: |
VERSION_CLASSIC=$(cat ./compiled/facebook-www/VERSION_CLASSIC)
VERSION_MODERN=$(cat ./compiled/facebook-www/VERSION_MODERN)
VERSION_NATIVE_FB=$(cat ./compiled-rn/VERSION_NATIVE_FB)
echo "Current classic version is $VERSION_CLASSIC"
echo "Current modern version is $VERSION_MODERN"
echo "Current rn version is $VERSION_NATIVE_FB"
echo "current_version_classic=$VERSION_CLASSIC" >> "$GITHUB_OUTPUT"
echo "current_version_modern=$VERSION_MODERN" >> "$GITHUB_OUTPUT"
echo "current_version_rn=$VERSION_NATIVE_FB" >> "$GITHUB_OUTPUT"
- uses: actions/upload-artifact@v3
with:
name: compiled
path: compiled/
- uses: actions/upload-artifact@v3
with:
name: compiled-rn
path: compiled-rn/
commit_www_artifacts:
needs: download_artifacts
if: ${{ (github.ref == 'refs/heads/main' && needs.download_artifacts.outputs.www_branch_count == '0') || github.ref == 'refs/heads/meta-www' }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
ref: builds/facebook-www
- name: Ensure clean directory
run: rm -rf compiled
- uses: actions/download-artifact@v3
with:
name: compiled
path: compiled/
- name: Revert version changes
if: needs.download_artifacts.outputs.last_version_classic != '' && needs.download_artifacts.outputs.last_version_modern != ''
env:
CURRENT_VERSION_CLASSIC: ${{ needs.download_artifacts.outputs.current_version_classic }}
CURRENT_VERSION_MODERN: ${{ needs.download_artifacts.outputs.current_version_modern }}
LAST_VERSION_CLASSIC: ${{ needs.download_artifacts.outputs.last_version_classic }}
LAST_VERSION_MODERN: ${{ needs.download_artifacts.outputs.last_version_modern }}
run: |
echo "Reverting $CURRENT_VERSION_CLASSIC to $LAST_VERSION_CLASSIC"
grep -rl "$CURRENT_VERSION_CLASSIC" ./compiled || echo "No files found with $CURRENT_VERSION_CLASSIC"
grep -rl "$CURRENT_VERSION_CLASSIC" ./compiled | xargs -r sed -i -e "s/$CURRENT_VERSION_CLASSIC/$LAST_VERSION_CLASSIC/g"
grep -rl "$CURRENT_VERSION_CLASSIC" ./compiled || echo "Classic version reverted"
echo "===================="
echo "Reverting $CURRENT_VERSION_MODERN to $LAST_VERSION_MODERN"
grep -rl "$CURRENT_VERSION_MODERN" ./compiled || echo "No files found with $CURRENT_VERSION_MODERN"
grep -rl "$CURRENT_VERSION_MODERN" ./compiled | xargs -r sed -i -e "s/$CURRENT_VERSION_MODERN/$LAST_VERSION_MODERN/g"
grep -rl "$CURRENT_VERSION_MODERN" ./compiled || echo "Modern version reverted"
- name: Check for changes
id: check_should_commit
run: |
echo "Full git status"
git add .
git status
echo "===================="
if git status --porcelain | grep -qv '/REVISION'; then
echo "Changes detected"
echo "===== Changes ====="
git --no-pager diff -U0 | grep '^[+-]' | head -n 50
echo "==================="
echo "should_commit=true" >> "$GITHUB_OUTPUT"
else
echo "No Changes detected"
echo "should_commit=false" >> "$GITHUB_OUTPUT"
fi
- name: Re-apply version changes
if: steps.check_should_commit.outputs.should_commit == 'true' && needs.download_artifacts.outputs.last_version_classic != '' && needs.download_artifacts.outputs.last_version_modern != ''
env:
CURRENT_VERSION_CLASSIC: ${{ needs.download_artifacts.outputs.current_version_classic }}
CURRENT_VERSION_MODERN: ${{ needs.download_artifacts.outputs.current_version_modern }}
LAST_VERSION_CLASSIC: ${{ needs.download_artifacts.outputs.last_version_classic }}
LAST_VERSION_MODERN: ${{ needs.download_artifacts.outputs.last_version_modern }}
run: |
echo "Re-applying $LAST_VERSION_CLASSIC to $CURRENT_VERSION_CLASSIC"
grep -rl "$LAST_VERSION_CLASSIC" ./compiled || echo "No files found with $LAST_VERSION_CLASSIC"
grep -rl "$LAST_VERSION_CLASSIC" ./compiled | xargs -r sed -i -e "s/$LAST_VERSION_CLASSIC/$CURRENT_VERSION_CLASSIC/g"
grep -rl "$LAST_VERSION_CLASSIC" ./compiled || echo "Classic version re-applied"
echo "===================="
echo "Re-applying $LAST_VERSION_MODERN to $CURRENT_VERSION_MODERN"
grep -rl "$LAST_VERSION_MODERN" ./compiled || echo "No files found with $LAST_VERSION_MODERN"
grep -rl "$LAST_VERSION_MODERN" ./compiled | xargs -r sed -i -e "s/$LAST_VERSION_MODERN/$CURRENT_VERSION_MODERN/g"
grep -rl "$LAST_VERSION_MODERN" ./compiled || echo "Classic version re-applied"
- name: Will commit these changes
if: steps.check_should_commit.outputs.should_commit == 'true'
run: |
echo ":"
git status -u
- name: Commit changes to branch
if: steps.check_should_commit.outputs.should_commit == 'true'
uses: stefanzweifel/git-auto-commit-action@v4
with:
commit_message: |
${{ github.event.head_commit.message }}
DiffTrain build for [${{ github.sha }}](https://github.com/facebook/react/commit/${{ github.sha }})
branch: builds/facebook-www
commit_user_name: ${{ github.actor }}
commit_user_email: ${{ github.actor }}@users.noreply.github.com
create_branch: true
commit_fbsource_artifacts:
needs: download_artifacts
if: ${{ (github.ref == 'refs/heads/main' && needs.download_artifacts.outputs.fbsource_branch_count == '0') || github.ref == 'refs/heads/meta-fbsource' }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
ref: builds/facebook-fbsource
- name: Ensure clean directory
run: rm -rf compiled-rn
- uses: actions/download-artifact@v3
with:
name: compiled-rn
path: compiled-rn/
- name: Revert version changes
if: needs.download_artifacts.outputs.last_version_rn != ''
env:
CURRENT_VERSION: ${{ needs.download_artifacts.outputs.current_version_rn }}
LAST_VERSION: ${{ needs.download_artifacts.outputs.last_version_rn }}
run: |
echo "Reverting $CURRENT_VERSION to $LAST_VERSION"
grep -rl "$CURRENT_VERSION" ./compiled-rn || echo "No files found with $CURRENT_VERSION"
grep -rl "$CURRENT_VERSION" ./compiled-rn | xargs -r sed -i -e "s/$CURRENT_VERSION/$LAST_VERSION/g"
grep -rl "$CURRENT_VERSION" ./compiled-rn || echo "Version reverted"
- name: Check changes before signing
run: |
echo "Full git status"
git add .
git status
echo "===================="
if git status --porcelain | grep -qv '/REVISION'; then
echo "Changes detected"
echo "===== Changes ====="
git --no-pager diff -U0 --cached | grep '^[+-]' | head -n 50
echo "==================="
else
echo "No Changes detected"
fi
- name: Revert signatures
uses: actions/github-script@v6
with:
script: |
// TODO: Move this to a script file.
// We currently can't call scripts from the repo because
// at this point in the workflow, we're on the compiled
// artifact branch (so the scripts don't exist).
// We can fix this with a composite action in the main repo.
// This script is duplicated below.
const fs = require('fs');
const crypto = require('crypto');
const {execSync} = require('child_process');
// TODO: when we move this to a script, we can use this from npm.
// Copy of signedsource since we can't install deps on this branch
const GENERATED = '@' + 'generated';
const NEWTOKEN = '<<SignedSource::*O*zOeWoEQle#+L!plEphiEmie@IsG>>';
const PATTERN = new RegExp(`${GENERATED} (?:SignedSource<<([a-f0-9]{32})>>)`);
const TokenNotFoundError = new Error(
`SignedSource.signFile(...): Cannot sign file without token: ${NEWTOKEN}`
);
function hash(data, encoding) {
const md5sum = crypto.createHash('md5');
md5sum.update(data, encoding);
return md5sum.digest('hex');
}
const SignedSource = {
getSigningToken() {
return `${GENERATED} ${NEWTOKEN}`;
},
isSigned(data) {
return PATTERN.exec(data) != null;
},
signFile(data) {
if (!data.includes(NEWTOKEN)) {
if (SignedSource.isSigned(data)) {
// Signing a file that was previously signed.
data = data.replace(PATTERN, SignedSource.getSigningToken());
} else {
throw TokenNotFoundError;
}
}
return data.replace(NEWTOKEN, `SignedSource<<${hash(data, 'utf8')}>>`);
},
};
const directory = './compiled-rn';
console.log('Signing files in directory:', directory);
try {
const result = execSync(`git status --porcelain ${directory}`, {encoding: 'utf8'});
// Parse the git status output to get file paths
const files = result.split('\n').filter(file => file.endsWith('.js'));
if (files.length === 0) {
throw new Error(
'git status returned no files to sign. this job should not have run.'
);
} else {
files.forEach(line => {
const file = line.slice(3).trim();
if (file) {
console.log(' Signing file:', file);
const originalContents = fs.readFileSync(file, 'utf8');
const signedContents = SignedSource.signFile(originalContents);
fs.writeFileSync(file, signedContents, 'utf8');
}
});
}
} catch (e) {
process.exitCode = 1;
console.error('Error signing files:', e);
}
- name: Check for changes
id: check_should_commit
run: |
echo "Full git status"
git add .
git status --porcelain
echo "===================="
if git status --porcelain | grep -qv '/REVISION'; then
echo "Changes detected"
echo "===== Changes ====="
git --no-pager diff -U0 --cached | grep '^[+-]' | head -n 50
echo "==================="
echo "should_commit=true" >> "$GITHUB_OUTPUT"
else
echo "No Changes detected"
echo "should_commit=false" >> "$GITHUB_OUTPUT"
fi
- name: Re-apply version changes
if: steps.check_should_commit.outputs.should_commit == 'true' && needs.download_artifacts.outputs.last_version_rn != ''
env:
CURRENT_VERSION: ${{ needs.download_artifacts.outputs.current_version_rn }}
LAST_VERSION: ${{ needs.download_artifacts.outputs.last_version_rn }}
run: |
echo "Re-applying $LAST_VERSION to $CURRENT_VERSION"
grep -rl "$LAST_VERSION" ./compiled-rn || echo "No files found with $LAST_VERSION"
grep -rl "$LAST_VERSION" ./compiled-rn | xargs -r sed -i -e "s/$LAST_VERSION/$CURRENT_VERSION/g"
grep -rl "$LAST_VERSION" ./compiled-rn || echo "Version re-applied"
- name: Add files
if: steps.check_should_commit.outputs.should_commit == 'true'
run: |
echo ":"
git add .
- name: Signing files
if: steps.check_should_commit.outputs.should_commit == 'true'
uses: actions/github-script@v6
with:
script: |
// TODO: Move this to a script file.
// We currently can't call scripts from the repo because
// at this point in the workflow, we're on the compiled
// artifact branch (so the scripts don't exist).
// We can fix this with a composite action in the main repo.
// This script is duplicated above.
const fs = require('fs');
const crypto = require('crypto');
const {execSync} = require('child_process');
// TODO: when we move this to a script, we can use this from npm.
// Copy of signedsource since we can't install deps on this branch.
const GENERATED = '@' + 'generated';
const NEWTOKEN = '<<SignedSource::*O*zOeWoEQle#+L!plEphiEmie@IsG>>';
const PATTERN = new RegExp(`${GENERATED} (?:SignedSource<<([a-f0-9]{32})>>)`);
const TokenNotFoundError = new Error(
`SignedSource.signFile(...): Cannot sign file without token: ${NEWTOKEN}`
);
function hash(data, encoding) {
const md5sum = crypto.createHash('md5');
md5sum.update(data, encoding);
return md5sum.digest('hex');
}
const SignedSource = {
getSigningToken() {
return `${GENERATED} ${NEWTOKEN}`;
},
isSigned(data) {
return PATTERN.exec(data) != null;
},
signFile(data) {
if (!data.includes(NEWTOKEN)) {
if (SignedSource.isSigned(data)) {
// Signing a file that was previously signed.
data = data.replace(PATTERN, SignedSource.getSigningToken());
} else {
throw TokenNotFoundError;
}
}
return data.replace(NEWTOKEN, `SignedSource<<${hash(data, 'utf8')}>>`);
},
};
const directory = './compiled-rn';
console.log('Signing files in directory:', directory);
try {
const result = execSync(`git status --porcelain ${directory}`, {encoding: 'utf8'});
// Parse the git status output to get file paths
const files = result.split('\n').filter(file => file.endsWith('.js'));
if (files.length === 0) {
throw new Error(
'git status returned no files to sign. this job should not have run.'
);
} else {
files.forEach(line => {
const file = line.slice(3).trim();
if (file) {
console.log(' Signing file:', file);
const originalContents = fs.readFileSync(file, 'utf8');
const signedContents = SignedSource.signFile(originalContents);
fs.writeFileSync(file, signedContents, 'utf8');
}
});
}
} catch (e) {
process.exitCode = 1;
console.error('Error signing files:', e);
}
- name: Will commit these changes
if: steps.check_should_commit.outputs.should_commit == 'true'
run: |
git status -u
- name: Commit changes to branch
if: steps.check_should_commit.outputs.should_commit == 'true'
uses: stefanzweifel/git-auto-commit-action@v4
with:
commit_message: |
${{ github.event.head_commit.message }}
DiffTrain build for commit https://github.com/facebook/react/commit/${{ github.sha }}.
branch: builds/facebook-fbsource
commit_user_name: ${{ github.actor }}
commit_user_email: ${{ github.actor }}@users.noreply.github.com
create_branch: true