deploy_job.sh

← Back to workflow

Builds, deploys, and schedules Cloud Run jobs.

#!/bin/bash
set -euo pipefail

# ----------------------------
# SETTINGS
# ----------------------------
PROJECT_ID="ggn-nmfs-swfscerddap-dev-1"
REGION="us-east4"

REPO_NAME="mur-daily-processor"
IMAGE_NAME="mur-daily-processor"
TAG="v1"

SERVICE_ACCOUNT="swfscerddap-run-sa1@${PROJECT_ID}.iam.gserviceaccount.com"

# If you're on Apple Silicon, keep this. Otherwise you can set PLATFORM=""
PLATFORM="${PLATFORM:-linux/amd64}"

# Full image path
IMAGE_PATH="${REGION}-docker.pkg.dev/${PROJECT_ID}/${REPO_NAME}/${IMAGE_NAME}:${TAG}"

# Secrets + envs shared by both jobs
COMMON_SECRETS="/secrets/netrc/file=netrc-secret:latest,/secrets/cookies/file=urs-cookies-secret:latest"
COMMON_ENVS="NETRC=/tmp/.netrc,URS_COOKIES=/tmp/.urs_cookies"

need() { command -v "$1" >/dev/null 2>&1 || { echo "Missing required command: $1" >&2; exit 1; }; }
need gcloud
need docker

echo "Using image: ${IMAGE_PATH}"

# ----------------------------
# 1) Build image
# ----------------------------
echo "Building Docker image..."
if [[ -n "${PLATFORM}" ]]; then
  docker buildx build --platform "${PLATFORM}" -t "${IMAGE_NAME}:${TAG}" --load .
else
  docker build -t "${IMAGE_NAME}:${TAG}" .
fi

# ----------------------------
# 2) gcloud project + Artifact Registry repo
# ----------------------------
echo "Setting gcloud project: ${PROJECT_ID}"
gcloud config set project "${PROJECT_ID}" >/dev/null

echo "Checking Artifact Registry repo: ${REPO_NAME} (${REGION})..."
if ! gcloud artifacts repositories describe "${REPO_NAME}" --location="${REGION}" >/dev/null 2>&1; then
  echo "Repo not found. Creating..."
  gcloud artifacts repositories create "${REPO_NAME}" \
    --repository-format=docker \
    --location="${REGION}" \
    --description="Docker repository for MUR daily processing"
else
  echo "Repo exists."
fi

echo "Configuring Docker auth for ${REGION}-docker.pkg.dev ..."
gcloud auth configure-docker "${REGION}-docker.pkg.dev" --quiet

# ----------------------------
# 3) Tag + push
# ----------------------------
echo "Tagging + pushing image..."
docker tag "${IMAGE_NAME}:${TAG}" "${IMAGE_PATH}"
docker push "${IMAGE_PATH}"
echo "OK: pushed ${IMAGE_PATH}"

# ----------------------------
# 4) Deploy Cloud Run Jobs (same image, different JOB_MODE)
# ----------------------------
echo "Deploying Cloud Run Jobs..."

deploy_job() {
  local job_name="$1"
  local job_mode="$2"
  local memory="$3"
  local cpu="$4"
  local timeout="$5"

  echo "Deploying job: ${job_name} (JOB_MODE=${job_mode})"
  gcloud run jobs deploy "${job_name}" \
    --image "${IMAGE_PATH}" \
    --service-account "${SERVICE_ACCOUNT}" \
    --tasks 1 \
    --max-retries 0 \
    --task-timeout "${timeout}" \
    --region "${REGION}" \
    --memory "${memory}" \
    --cpu "${cpu}" \
    --set-secrets "${COMMON_SECRETS}" \
    --set-env-vars "${COMMON_ENVS},JOB_MODE=${job_mode}"
}

deploy_job "mur41-daily-processor"   "mur41"         "32Gi" "8" "1h30m"
deploy_job "mur42-daily-processor"   "mur42"         "8Gi"  "4" "1h"
deploy_job "mur41-monthly-processor" "mur41_monthly" "32Gi" "8" "3h"

# ----------------------------
# 5) Create Cloud Scheduler jobs to run Cloud Run Jobs
# ----------------------------
echo "Configuring Cloud Scheduler..."

PROJECT_NUMBER="$(gcloud projects describe "${PROJECT_ID}" --format='value(projectNumber)')"

create_scheduler() {
  local scheduler_name="$1"
  local job_name="$2"
  local cron="$3"
  local tz="$4"

  echo "Creating scheduler: ${scheduler_name} -> job: ${job_name} (${cron} ${tz})"

  gcloud scheduler jobs delete "${scheduler_name}" --location "${REGION}" --quiet || true

  gcloud scheduler jobs create http "${scheduler_name}" \
    --location "${REGION}" \
    --schedule "${cron}" \
    --time-zone "${tz}" \
    --uri "https://${REGION}-run.googleapis.com/apis/run.googleapis.com/v1/namespaces/${PROJECT_NUMBER}/jobs/${job_name}:run" \
    --http-method POST \
    --oauth-service-account-email "${SERVICE_ACCOUNT}" \
    --oauth-token-scope "https://www.googleapis.com/auth/cloud-platform"
}

# Run at top of hour, every 4 hours
create_scheduler "mur41-daily-processor"   "mur41-daily-processor"   "0 0,4,8,12,16,20 * * *" "America/Los_Angeles"

# Run 45 minutes after mur41
create_scheduler "mur42-daily-processor"   "mur42-daily-processor"   "45 0,4,8,12,16,20 * * *" "America/Los_Angeles"

# Run on the 3rd of each month at 13:00 UTC
create_scheduler "mur41-monthly-processor" "mur41-monthly-processor" "0 13 3 * *"               "America/Los_Angeles"

echo "All done."
echo "Image: ${IMAGE_PATH}"
echo "Jobs:       mur41-daily-processor, mur42-daily-processor, mur41-monthly-processor"
echo "Schedulers: mur41-daily-processor, mur42-daily-processor, mur41-monthly-processor"