release: init action to release first party PRs (#162)

* release: init action to release first party PRs

* release: check if revision has already been uploaded

We don't want to waste bandwidth on files that already exist.

* release: consolidate s3 uploading into single script; check if artifact already uploaded

* release: only upload release for PR if it has the "upload to s3" label

.github/workflows/release-branches.yml

@@ -26,24 +26,8 @@ jobs:
           role-to-assume: ${{ secrets.AWS_S3_UPLOAD_ROLE }}
           aws-region: us-east-2
       - name: Publish Release (Branch)
+        env:
+          AWS_BUCKET: ${{ secrets.AWS_S3_UPLOAD_BUCKET }}
         run: |
-          sudo chown $USER: -R artifacts/
-
           LATEST_BRANCH="latest_${{ github.ref_name }}"
-          mkdir "$LATEST_BRANCH"
-          mkdir "$GITHUB_SHA"
-
-          sed -i "s@https://install.determinate.systems/nix@https://install.determinate.systems/nix/rev/$GITHUB_SHA@" nix-installer.sh
-          cp nix-installer.sh "$GITHUB_SHA"/
-          cp nix-installer.sh "$LATEST_BRANCH"/
-
-          for artifact in $(find artifacts/ -type f); do
-            chmod +x "$artifact"
-            cp "$artifact" "$GITHUB_SHA"/
-            cp "$artifact" "$LATEST_BRANCH"/
-          done
-
-          # TODO: determine if these binaries have already been uploaded / are the exact same (try download and then hash if already exists as latest_*?)
-
-          aws s3 sync "$GITHUB_SHA"/ s3://${{ secrets.AWS_S3_UPLOAD_BUCKET }}/"$GITHUB_SHA"/ --acl public-read
-          aws s3 sync "$LATEST_BRANCH"/ s3://${{ secrets.AWS_S3_UPLOAD_BUCKET }}/"$LATEST_BRANCH"/ --acl public-read
+          ./upload_s3.sh "$LATEST_BRANCH"

.github/workflows/release-prs.yml

@@ -0,0 +1,36 @@
+name: Release PR
+
+on:
+  pull_request:
+    types:
+      - opened
+      - reopened
+      - synchronize
+      - labeled
+
+jobs:
+  release:
+    # Only intra-repo PRs are allowed to have PR artifacts uploaded
+    if: github.event.pull_request.head.repo.full_name == 'DeterminateSystems/nix-installer' && contains(github.event.pull_request.labels.*.name, 'upload to s3')
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write # In order to request a JWT for AWS auth
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Download Buildkite Artifacts
+        uses: EnricoMi/download-buildkite-artifact-action@v1.14
+        with:
+          buildkite_token: ${{ secrets.BUILDKITE_TOKEN }}
+          output_path: artifacts
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          role-to-assume: ${{ secrets.AWS_S3_UPLOAD_ROLE }}
+          aws-region: us-east-2
+      - name: Publish Release (PR)
+        env:
+          AWS_BUCKET: ${{ secrets.AWS_S3_UPLOAD_BUCKET }}
+        run: |
+          PR="pr_${{ github.event.pull_request.number }}"
+          ./upload_s3.sh "$PR"

upload_s3.sh

@@ -0,0 +1,37 @@
+set -eu
+
+# If the revision directory has already been created in S3 somehow, we don't want to reupload
+if aws s3 ls "$AWS_BUCKET"/"$GITHUB_SHA"/; then
+  echo "Revision $GITHUB_SHA was already uploaded; exiting"
+  exit 1
+fi
+
+sudo chown $USER: -R artifacts/
+
+DEST="$1"
+
+mkdir "$GITHUB_SHA"
+
+sed -i "s@https://install.determinate.systems/nix@https://install.determinate.systems/nix/rev/$GITHUB_SHA@" nix-installer.sh
+cp nix-installer.sh "$GITHUB_SHA"/
+
+for artifact in $(find artifacts/ -type f); do
+  chmod +x "$artifact"
+  cp "$artifact" "$GITHUB_SHA"/
+done
+
+# If any artifact already exists in S3 and the hash is the same, we don't want to reupload
+for file in $(find "$GITHUB_SHA" -type f); do
+  artifact_path="$DEST"/"$(basename "$artifact")"
+  md5="$(md5sum "$artifact" | cut -d' ' -f1)"
+  obj="$(aws s3api head-object --bucket "$AWS_BUCKET" --key "$artifact_path")"
+  obj_md5="$(jq -r .ETag <<<"$obj" | jq -r)" # head-object call returns ETag quoted, so `jq -r` again to unquote it
+
+  if [[ "$md5" == "$obj_md5" ]]; then
+    echo "Artifact $artifact was already uploaded; exiting"
+    exit 1
+  fi
+done
+
+aws s3 sync "$GITHUB_SHA"/ s3://"$AWS_BUCKET"/"$GITHUB_SHA"/ --acl public-read
+aws s3 sync "$GITHUB_SHA"/ s3://"$AWS_BUCKET"/"$DEST"/ --acl public-read