upload_s3: improve tags handling (#202)

This commit is contained in:
Cole Helbling 2023-01-20 09:42:14 -08:00 committed by GitHub
parent 24a94ec3f9
commit bb4ef37ea5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 51 additions and 21 deletions

View file

@ -31,4 +31,4 @@ jobs:
run: | run: |
BRANCH="branch_${{ github.ref_name }}" BRANCH="branch_${{ github.ref_name }}"
GIT_ISH="$GITHUB_SHA" GIT_ISH="$GITHUB_SHA"
./upload_s3.sh "$BRANCH" "$GIT_ISH" ./upload_s3.sh "$BRANCH" "$GIT_ISH" "https://install.determinate.systems/nix/rev/$GIT_ISH"

View file

@ -34,4 +34,4 @@ jobs:
run: | run: |
PR="pr_${{ github.event.pull_request.number }}" PR="pr_${{ github.event.pull_request.number }}"
GIT_ISH="${{ github.event.pull_request.head.sha }}" GIT_ISH="${{ github.event.pull_request.head.sha }}"
./upload_s3.sh "$PR" "$GIT_ISH" ./upload_s3.sh "$PR" "$GIT_ISH" "https://install.determinate.systems/nix/rev/$GIT_ISH"

View file

@ -19,9 +19,6 @@ jobs:
with: with:
buildkite_token: ${{ secrets.BUILDKITE_TOKEN }} buildkite_token: ${{ secrets.BUILDKITE_TOKEN }}
output_path: artifacts output_path: artifacts
- name: Fixup URL in nix-installer.sh
run: |
sed -i "s@https://install.determinate.systems/nix@https://install.determinate.systems/nix/tag/$GITHUB_REF_NAME@" nix-installer.sh
- name: Configure AWS Credentials - name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v1 uses: aws-actions/configure-aws-credentials@v1
with: with:
@ -31,7 +28,7 @@ jobs:
env: env:
AWS_BUCKET: ${{ secrets.AWS_S3_UPLOAD_BUCKET }} AWS_BUCKET: ${{ secrets.AWS_S3_UPLOAD_BUCKET }}
run: | run: |
./upload_s3.sh "$GITHUB_REF_NAME" "$GITHUB_SHA" ./upload_s3.sh "$GITHUB_REF_NAME" "$GITHUB_SHA" "https://install.determinate.systems/nix/tag/$GITHUB_REF_NAME"
- name: Publish Release to GitHub (Tag) - name: Publish Release to GitHub (Tag)
uses: softprops/action-gh-release@v1 uses: softprops/action-gh-release@v1
with: with:

View file

@ -2,37 +2,70 @@ set -eu
DEST="$1" DEST="$1"
GIT_ISH="$2" GIT_ISH="$2"
DEST_INSTALL_URL="$3"
is_tag() {
if [[ "$GITHUB_REF_TYPE" == "tag" ]]; then
return 0
else
return 1
fi
}
# If the revision directory has already been created in S3 somehow, we don't want to reupload # If the revision directory has already been created in S3 somehow, we don't want to reupload
if aws s3 ls "$AWS_BUCKET"/"$GIT_ISH"/; then if aws s3 ls "$AWS_BUCKET"/"$GIT_ISH"/; then
# Only exit if it's not a tag (since we're tagging a commit previously pushed to main)
if ! is_tag; then
echo "Revision $GIT_ISH was already uploaded; exiting" echo "Revision $GIT_ISH was already uploaded; exiting"
exit 1 exit 1
fi
fi fi
sudo chown $USER: -R artifacts/ sudo chown $USER: -R artifacts/
mkdir "$DEST"
mkdir "$GIT_ISH" mkdir "$GIT_ISH"
sed -i "s@https://install.determinate.systems/nix@https://install.determinate.systems/nix/rev/$GIT_ISH@" nix-installer.sh cp nix-installer.sh "$DEST"/
cp nix-installer.sh "$GIT_ISH"/ cp nix-installer.sh "$GIT_ISH"/
for artifact in $(find artifacts/ -type f); do for artifact in $(find artifacts/ -type f); do
chmod +x "$artifact" chmod +x "$artifact"
cp "$artifact" "$DEST"/
cp "$artifact" "$GIT_ISH"/ cp "$artifact" "$GIT_ISH"/
done done
sed -i "s@https://install.determinate.systems/nix@$DEST_INSTALL_URL@" "$DEST/nix-installer.sh"
sed -i "s@https://install.determinate.systems/nix@https://install.determinate.systems/nix/rev/$GIT_ISH@" "$GIT_ISH/nix-installer.sh"
if is_tag; then
cp "$DEST/nix-installer.sh" ./nix-installer.sh
fi
# If any artifact already exists in S3 and the hash is the same, we don't want to reupload # If any artifact already exists in S3 and the hash is the same, we don't want to reupload
for file in $(find "$GIT_ISH" -type f); do check_reupload() {
artifact_path="$DEST"/"$(basename "$artifact")" dest="$1"
for file in $(find "$dest" -type f); do
artifact_path="$dest"/"$(basename "$artifact")"
md5="$(md5sum "$artifact" | cut -d' ' -f1)" md5="$(md5sum "$artifact" | cut -d' ' -f1)"
obj="$(aws s3api head-object --bucket "$AWS_BUCKET" --key "$artifact_path" || echo '{}')" obj="$(aws s3api head-object --bucket "$AWS_BUCKET" --key "$artifact_path" || echo '{}')"
obj_md5="$(jq -r .ETag <<<"$obj" | jq -r)" # head-object call returns ETag quoted, so `jq -r` again to unquote it obj_md5="$(jq -r .ETag <<<"$obj" | jq -r)" # head-object call returns ETag quoted, so `jq -r` again to unquote it
if [[ "$md5" == "$obj_md5" ]]; then if [[ "$md5" == "$obj_md5" ]]; then
echo "Artifact $artifact was already uploaded; exiting" echo "Artifact $artifact was already uploaded; exiting"
exit 0 # If we already uploaded to a tag, that's probably bad
is_tag && exit 1 || exit 0
fi fi
done done
}
aws s3 sync "$GIT_ISH"/ s3://"$AWS_BUCKET"/"$GIT_ISH"/ --acl public-read check_reupload "$DEST"
aws s3 sync s3://"$AWS_BUCKET"/"$GIT_ISH"/ s3://"$AWS_BUCKET"/"$DEST"/ --acl public-read if ! is_tag; then
check_reupload "$GIT_ISH"
fi
aws s3 sync "$DEST"/ s3://"$AWS_BUCKET"/"$DEST"/ --acl public-read
if ! is_tag; then
aws s3 sync "$GIT_ISH"/ s3://"$AWS_BUCKET"/"$GIT_ISH"/ --acl public-read
fi