config.yml

← Back to workflow

Runtime configuration — GCS buckets, NASA endpoints, publish targets.

# ============================================================
# Configuration for running ERD processing scripts (GCP)
# ------------------------------------------------------------
# Some considerations:
#  - Work bucket is NOT mounted on the dataproc VM (always access via GCS APIs).
#  - Production/ERDDAP bucket is served (mounted continuously) on the ERDDAP VM, but dataproc
#    should still upload to prod via GCS APIs.
#  - Publishing destinations are declared under PUBLISH_TARGETS to avoid
#    hard-coded GCS object layouts inside code.
# ============================================================

# ------------------------------------------------------------
# Root directories (local filesystem on the dataproc VM)
# ------------------------------------------------------------
HOME_DIR: "/tmp"                        # Project home on dataproc VM

# Core subdirectories under HOME_DIR
# NOTE:
#  - These are relative paths. Code should join them to HOME_DIR.
DIRS:
  CDL_DIR: "templates"                       # CDL/templates used by ncgen
  WORK_MW_DIR: "data/work_mw"                # MW (West Coast) processing workspace
  WORK_MB_DIR: "data/work_mb"                # MB (Pacific Ocean) processing workspace
  LOG_DIR: "logs"                            # Directory for log file output

  # Local staging (since WORK bucket is not mounted)
  LOCAL_STAGE_DIR: "data/staging"            # Scratch + downloads + intermediate files
  LOCAL_L2_DIR: "data/staging/L2"            # Optional local staging for L2 swaths
  LOCAL_OUTPUT_DIR: "data/staging/out"       # Optional local staging for final outputs before upload

# ------------------------------------------------------------
# Google Cloud Storage buckets (names, not gs:// URIs)
# ------------------------------------------------------------
# NOTE:
#  - Bucket names are used by google-cloud-storage and gcsfs.
#  - Do NOT include the "gs://" prefix.
ERDWORK_BUCKET: "YOUR_WORK_BUCKET"             # Work/staging bucket (L2 downloads, temps)
ERDPROD_BUCKET: "YOUR_PRODUCTION_BUCKET"           # Production bucket (served by ERDDAP/THREDDS)

# ------------------------------------------------------------
# Work-bucket object layout (bucket-relative prefixes)
# ------------------------------------------------------------
# These are "folders" inside the WORK bucket.
# Use these to build:
#   gs://<work_bucket>/<work_l2_prefix>/<YYYY>/<DOY>/OC/*.nc
WORK_L2_PREFIX: "modisa/data/L2"             # L2 swath store root in the WORK bucket

# Convenience prefixes for legacy-style daily mirrors
WORK_DAILY_GF_PREFIX: "modisa/data/modisgf/1day"  # MB 1-day legacy tree
WORK_DAILY_WC_PREFIX: "modisa/data/modiswc/1day"  # MW 1-day legacy tree

# ============================================================
# Publishing targets (PROD + WORK copy)
# ------------------------------------------------------------
# Logic implemented in send_to_servers():
#
# 1) Always upload to PROD:
#    gs://<prod_bucket>/<prod_root>/<dst_dir>/<interval>day/<filename>
#
# 2) Additionally, when:
#      - interval == "1"
#      - parent of <dst_dir> is "MB" or "MW"
#
#    Upload a second copy to WORK:
#    gs://<work_bucket>/<work_root>/<region>/1day/<filename>
# ============================================================

PUBLISH_TARGETS:
  prod:
    bucket: "YOUR_PRODUCTION_BUCKET"        # Production bucket (served by ERDDAP/THREDDS)
    root: "satellite"                 # Object prefix inside PROD bucket
    interval_fmt: "{interval}day"     # "{interval}" -> "0", "1", "3", etc.

  work:
    bucket: "YOUR_WORK_BUCKET"          # Work / staging bucket
    root: "modisa/data"               # Object prefix inside WORK bucket
    interval_fmt: "{interval}day"     # "{interval}" -> "0", "1", "3", etc.
    
    # Mapping from region code (parent of dst_dir)
    # to subdirectory name under WORK root
    # Examples:
    #   dst_dir = "MB/chla" → parent "MB" → modisgf/1day
    #   dst_dir = "MW/par0" → parent "MW" → modiswc/1day
    region_map:
      MB: "modisgf"
      MW: "modiswc"


# ------------------------------------------------------------
# Network sources & auxiliary endpoints
# ------------------------------------------------------------
NASA_OCEANDATA_URL: "https://oceandata.sci.gsfc.nasa.gov"
NASA_FILE_SEARCH_URL: "https://oceandata.sci.gsfc.nasa.gov/file_search"
NASA_GETFILE_URL: "https://oceandata.sci.gsfc.nasa.gov/ob/getfile"