diff --git a/.envExample b/.envExample index 66d429a..573c064 100755 --- a/.envExample +++ b/.envExample @@ -1,3 +1,7 @@ +################ +## Infra variables +################ + AIRFLOW_UID=1000 POSTGRES_USER= POSTGRES_PASSWORD= @@ -15,5 +19,21 @@ AIRFLOW_VAR__METADATA_DB_SCHEMA=$POSTGRES_DB AIRFLOW_ENV=dev AIRFLOW_DAG_HOME=/opt/airflow/dags/ AIRFLOW_DAG_TMP=/tmp/ -DATAGOUV_URL=https://demo.data.gouv.fr -DATAGOUV_SECRET_API_KEY= +AIRFLOW_PROJ_DIR=. +AIRFLOW_DAGS_DIR=. + +################ +## DAG variables +################ + +### S3 +AIRFLOW_VAR_S3_BUCKET_DATA_PIPELINE_OPEN= +AIRFLOW_VAR_SECRET_S3_USER= +AIRFLOW_VAR_SECRET_S3_PASSWORD= +AIRFLOW_VAR_S3_URL_RBX= +AIRFLOW_VAR_S3_URL_SBG= + +### Datagouv +AIRFLOW_VAR_DATAGOUV_URL=https://demo.data.gouv.fr +AIRFLOW_VAR_DATAGOUV_SECRET_API_KEY= +AIRFLOW_VAR_DEMO_DATAGOUV_SECRET_API_KEY= diff --git a/docker-compose.yml b/docker-compose.yml index bbeecca..e181de4 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -75,10 +75,11 @@ x-airflow-common: # The following line can be used to set a custom config file, stored in the local config folder AIRFLOW_CONFIG: '/opt/airflow/config/airflow.cfg' volumes: - - ${AIRFLOW_PROJ_DIR:-.}/dags:/opt/airflow/dags + - ${AIRFLOW_DAGS_DIR:-.}:/opt/airflow/dags/datagouvfr_data_pipelines - ${AIRFLOW_PROJ_DIR:-.}/logs:/opt/airflow/logs - ${AIRFLOW_PROJ_DIR:-.}/config:/opt/airflow/config - ${AIRFLOW_PROJ_DIR:-.}/plugins:/opt/airflow/plugins + - ${AIRFLOW_PROJ_DIR:-.}/tmp:${AIRFLOW_DAG_TMP:-/tmp/} user: "${AIRFLOW_UID:-50000}:0" depends_on: &airflow-common-depends-on @@ -291,8 +292,8 @@ services: <<: *airflow-common-env _AIRFLOW_DB_MIGRATE: 'true' _AIRFLOW_WWW_USER_CREATE: 'true' - _AIRFLOW_WWW_USER_USERNAME: ${_AIRFLOW_WWW_USER_USERNAME} - _AIRFLOW_WWW_USER_PASSWORD: ${_AIRFLOW_WWW_USER_PASSWORD} + _AIRFLOW_WWW_USER_USERNAME: ${_AIRFLOW_WWW_USER_USERNAME:-airflow} + _AIRFLOW_WWW_USER_PASSWORD: ${_AIRFLOW_WWW_USER_PASSWORD:-airflow} _PIP_ADDITIONAL_REQUIREMENTS: '' user: "0:0"