diff --git a/.release-notes.md b/.release-notes.md index 55849eba..5bd9bef7 100644 --- a/.release-notes.md +++ b/.release-notes.md @@ -1,21 +1,12 @@ -## LinuxAid Release Version v1.8.1 - -### Features -- cf5f770a feat: support for read only roles in role::storage::s3 -- 60d31222 feat: publish running slurm version for skew detection +## LinuxAid Release Version v1.8.2 ### Bug Fixes -- c198beaf fix: correct netcat and vim package names for SUSE and RedHat -- 90c85aa7 fix(repository): exclude EFI,isolinux,images from rocky mirror, use CSV format for EXCLUDE -- ef86395b fix: enable certificate monitoring with acme for http only domains -- b7e63260 fix: restore SECTIONS template to master format -- a9304e15 fix: remove SECTIONS line from template -- ab054d6b fix: render SECTIONS as bash array instead of CSV string -- 1872ddcf fix: remove sections validation for RPM repos -- 8ab08e28 fix: validate sections against allowed list for RPM repos -- 862417ea fix: allow multiple sections for RPM repos (e.g. Rocky Linux BaseOS, AppStream, extras) -- 582ac5e5 fix: add Rocky Linux yum sections (BaseOS, AppStream, extras) to mirrors config - -### Configuration Changes -- feff6d83 chore: update linuxaid-cli package v1.6.0 checksums +- bc4452d9 fix(repository): force repository_mirror_sync timer to activate under --noop +- 5515af03 fix: role::storage::s3 s3 service in docker compose and init policy script +- 51558019 fix: patched monitoring-based manifests to work properly with linuxaid_enc.rb + +### Other Changes +- c01574ef add timeout variable in obmondo security exporter instead of hardcoded value +- 8e910539 monitoring: configure zfs exporter via appropriate packages +- 1525b211 added kubeaid role diff --git a/CHANGELOG.md b/CHANGELOG.md index a9639c09..23ca9005 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,18 @@ All releases and the changes included in them (pulled from git commits added since last release) will be detailed in this file. +## LinuxAid Release Version v1.8.2 + +### Bug Fixes +- bc4452d9 fix(repository): force repository_mirror_sync timer to activate under --noop +- 5515af03 fix: role::storage::s3 s3 service in docker compose and init policy script +- 51558019 fix: patched monitoring-based manifests to work properly with linuxaid_enc.rb + +### Other Changes +- c01574ef add timeout variable in obmondo security exporter instead of hardcoded value +- 8e910539 monitoring: configure zfs exporter via appropriate packages +- 1525b211 added kubeaid role + ## LinuxAid Release Version v1.8.1 ### Features diff --git a/hiera.yaml b/hiera.yaml index 989092ee..91984a2a 100644 --- a/hiera.yaml +++ b/hiera.yaml @@ -4,7 +4,7 @@ version: 5 defaults: # relative path as per puppetserver on k8s, defaulting to main branch # in case of opensource the customer_id will be empty - datadir: ../../hiera-data/%{hiera_datapath} + datadir: ../../hiera-data/enableit/feat_s3_write_only_roles data_hash: yaml_data hierarchy: diff --git a/modules/enableit/common/data/common.yaml b/modules/enableit/common/data/common.yaml index 9fcc6e04..01dc453a 100644 --- a/modules/enableit/common/data/common.yaml +++ b/modules/enableit/common/data/common.yaml @@ -312,7 +312,7 @@ common::system::disabled_services: # Puppet/Openvox common::system::openvox::version: "8.24.2" -common::system::openvox::environment: "v1.8.1" +common::system::openvox::environment: "v1.8.2" common::system::openvox::server: enableit.puppet.obmondo.com common::system::openvox::package_name: "openvox-agent" common::system::openvox::configure_agent: true diff --git a/modules/enableit/eit_types/types/storage/s3/writeonly.pp b/modules/enableit/eit_types/types/storage/s3/writeonly.pp new file mode 100644 index 00000000..7b8c269b --- /dev/null +++ b/modules/enableit/eit_types/types/storage/s3/writeonly.pp @@ -0,0 +1,8 @@ +type Eit_types::Storage::S3::WriteOnly = Hash[String, Struct[{ + email => Eit_types::Email, + access_key => String, + buckets => Array[Struct[{ + bucket => String, + owner => String, + }]], +}]] diff --git a/modules/enableit/profile/manifests/storage/s3.pp b/modules/enableit/profile/manifests/storage/s3.pp index a1c4ad1d..39627b6b 100644 --- a/modules/enableit/profile/manifests/storage/s3.pp +++ b/modules/enableit/profile/manifests/storage/s3.pp @@ -9,7 +9,8 @@ Eit_types::Storage::S3 $roles = $role::storage::s3::roles, - Eit_types::Storage::S3::ReadOnly $read_only_roles = $role::storage::s3::read_only_roles, + Eit_types::Storage::S3::ReadOnly $read_only_roles = $role::storage::s3::read_only_roles, + Eit_types::Storage::S3::WriteOnly $write_only_roles = $role::storage::s3::write_only_roles, ) { # TODO: function does not work as it should @@ -54,7 +55,26 @@ } } - $_accounts = $_admin_accounts + $_readonly_accounts + $_writeonly_accounts = $write_only_roles.reduce([]) |$acc, $role| { + [$_name, $opts] = $role + + $_shortid = 123456789013 + seeded_rand(999, "${opts['email']}_${_name}") + $_arn = "arn:aws:iam::${_shortid}:root" + + $acc << { + name => $_name, + canonicalID => seeded_rand_string(65, "${opts['email']}_${_name}"), + email => $opts['email'], + arn => $_arn, + shortid => String($_shortid), + keys => [{ + 'access' => $_name, + 'secret' => $opts['access_key'], + }] + } + } + + $_accounts = $_admin_accounts + $_readonly_accounts + $_writeonly_accounts $_admin_creds = $roles.map |$_name, $_opts| { { 'accessKey' => $_name, 'secretKey' => $_opts['access_key'] } @@ -64,12 +84,20 @@ { 'accessKey' => $_name, 'secretKey' => $_opts['access_key'] } } + $_writeonly_creds = $write_only_roles.map |$_name, $_opts| { + { 'accessKey' => $_name, 'secretKey' => $_opts['access_key'] } + } + $_readonly_canonical_ids = $read_only_roles.map |$_name, $_opts| { seeded_rand_string(65, "${_opts['email']}_${_name}") } + $_writeonly_canonical_ids = $write_only_roles.map |$_name, $_opts| { + seeded_rand_string(65, "${_opts['email']}_${_name}") + } + # Build per-bucket read-only policies grouped by (bucket, owner) - $_raw_policies = $read_only_roles.reduce([]) |$acc, $role| { + $_raw_ro_policies = $read_only_roles.reduce([]) |$acc, $role| { [$_ro_name, $_ro_opts] = $role $_ro_canonical_id = seeded_rand_string(65, "${_ro_opts['email']}_${_ro_name}") $_ro_shortid = 123456789013 + seeded_rand(999, "${_ro_opts['email']}_${_ro_name}") @@ -77,15 +105,16 @@ $_entries = $_ro_opts['buckets'].reduce([]) |$inner_acc, $be| { if $be['owner'] in $roles { $inner_acc << { - 'bucket' => $be['bucket'], - 'ownerAccessKey' => $be['owner'], - 'ownerSecretKey' => $roles[$be['owner']]['access_key'], - 'readOnlyAccounts' => [{ + 'bucket' => $be['bucket'], + 'ownerAccessKey' => $be['owner'], + 'ownerSecretKey' => $roles[$be['owner']]['access_key'], + 'readOnlyAccounts' => [{ 'accessKey' => $_ro_name, 'secretKey' => $_ro_opts['access_key'], 'canonicalId' => $_ro_canonical_id, 'shortid' => String($_ro_shortid), }], + 'writeOnlyAccounts' => [], } } else { warning("S3 bucket policy: owner '${be['owner']}' not found in roles, skipping bucket '${be['bucket']}' for read-only role '${$_ro_name}'") @@ -96,15 +125,47 @@ $acc + $_entries } + # Build per-bucket write-only (no read/delete) policies grouped by (bucket, owner) + $_raw_wo_policies = $write_only_roles.reduce([]) |$acc, $role| { + [$_wo_name, $_wo_opts] = $role + $_wo_canonical_id = seeded_rand_string(65, "${_wo_opts['email']}_${_wo_name}") + $_wo_shortid = 123456789013 + seeded_rand(999, "${_wo_opts['email']}_${_wo_name}") + + $_entries = $_wo_opts['buckets'].reduce([]) |$inner_acc, $be| { + if $be['owner'] in $roles { + $inner_acc << { + 'bucket' => $be['bucket'], + 'ownerAccessKey' => $be['owner'], + 'ownerSecretKey' => $roles[$be['owner']]['access_key'], + 'readOnlyAccounts' => [], + 'writeOnlyAccounts' => [{ + 'accessKey' => $_wo_name, + 'secretKey' => $_wo_opts['access_key'], + 'canonicalId' => $_wo_canonical_id, + 'shortid' => String($_wo_shortid), + }], + } + } else { + warning("S3 bucket policy: owner '${be['owner']}' not found in roles, skipping bucket '${be['bucket']}' for write-only role '${$_wo_name}'") + $inner_acc + } + } + + $acc + $_entries + } + + $_raw_policies = $_raw_ro_policies + $_raw_wo_policies + $_bucket_policies = $_raw_policies.reduce({}) |$map, $entry| { $_key = "${entry['bucket']}|${entry['ownerAccessKey']}" if $_key in $map { $_existing = $map[$_key] $map + { $_key => { - 'bucket' => $_existing['bucket'], - 'ownerAccessKey' => $_existing['ownerAccessKey'], - 'ownerSecretKey' => $_existing['ownerSecretKey'], - 'readOnlyAccounts' => $_existing['readOnlyAccounts'] + $entry['readOnlyAccounts'], + 'bucket' => $_existing['bucket'], + 'ownerAccessKey' => $_existing['ownerAccessKey'], + 'ownerSecretKey' => $_existing['ownerSecretKey'], + 'readOnlyAccounts' => $_existing['readOnlyAccounts'] + $entry['readOnlyAccounts'], + 'writeOnlyAccounts' => $_existing['writeOnlyAccounts'] + $entry['writeOnlyAccounts'], }} } else { $map + { $_key => $entry } @@ -143,10 +204,12 @@ "${conf_dir}/s3-sideloader-config.json": ensure => file, content => stdlib::to_json_pretty({ - adminAccounts => $_admin_creds, - readOnlyAccounts => $_readonly_creds, - readOnlyCanonicalIds => $_readonly_canonical_ids, - bucketPolicies => $_bucket_policies, + adminAccounts => $_admin_creds, + readOnlyAccounts => $_readonly_creds, + readOnlyCanonicalIds => $_readonly_canonical_ids, + writeOnlyAccounts => $_writeonly_creds, + writeOnlyCanonicalIds => $_writeonly_canonical_ids, + bucketPolicies => $_bucket_policies, }).node_encrypt::secret, ; '/opt/obmondo/docker-compose/s3/s3-policy-init.sh': diff --git a/modules/enableit/profile/templates/docker-compose/s3/s3-policy-init.sh.epp b/modules/enableit/profile/templates/docker-compose/s3/s3-policy-init.sh.epp index 470cd046..17df92de 100644 --- a/modules/enableit/profile/templates/docker-compose/s3/s3-policy-init.sh.epp +++ b/modules/enableit/profile/templates/docker-compose/s3/s3-policy-init.sh.epp @@ -27,12 +27,22 @@ apply_policies() { owner_sk=$(echo "$policy" | jq -r '.ownerSecretKey') POLICY=$(echo "$policy" | jq --arg bucket "$bucket" '{ "Version": "2012-10-17", - "Statement": [.readOnlyAccounts[] | { + "Statement": ([.readOnlyAccounts[] | { "Effect": "Allow", "Principal": {"AWS": [.shortid]}, "Action": ["s3:GetObject", "s3:ListBucket"], "Resource": ["arn:aws:s3:::" + $bucket, "arn:aws:s3:::" + $bucket + "/*"] - }] + }] + [.writeOnlyAccounts[] | { + "Effect": "Allow", + "Principal": {"AWS": [.shortid]}, + "Action": ["s3:PutObject"], + "Resource": ["arn:aws:s3:::" + $bucket + "/*"] + }] + [.writeOnlyAccounts[] | { + "Effect": "Allow", + "Principal": {"AWS": [.shortid]}, + "Action": ["s3:ListBucket"], + "Resource": ["arn:aws:s3:::" + $bucket] + }]) }') export AWS_ACCESS_KEY_ID="$owner_ak" diff --git a/modules/enableit/role/manifests/storage/s3.pp b/modules/enableit/role/manifests/storage/s3.pp index 12e7c343..25b173ab 100644 --- a/modules/enableit/role/manifests/storage/s3.pp +++ b/modules/enableit/role/manifests/storage/s3.pp @@ -11,7 +11,40 @@ # # @param roles The S3 storage roles. # -# @param read_only_roles The S3 read-only storage roles. +# @param read_only_roles +# The S3 read-only storage roles. Each entry's `buckets[].owner` must match +# the name of an account declared in `$roles`, otherwise that bucket grant +# is skipped with a warning. +# +# @example Hiera: read-only access to buckets owned by existing $roles accounts +# role::storage::s3::read_only_roles: +# backup-exporter: +# access_key: ENC[PKCS7,...] +# email: backup-exporter@example.com +# buckets: +# - bucket: app-logs-backup +# owner: app +# - bucket: app-db-backup +# owner: app +# +# @param write_only_roles +# The S3 write-only (no read/delete) storage roles. Same shape as +# `$read_only_roles` — each entry's `buckets[].owner` must match an +# account declared in `$roles`. Grants `s3:PutObject` on the listed +# buckets plus `s3:ListBucket` on the bucket itself (needed by clients, +# e.g. replication tools, that check bucket existence before uploading); +# the account cannot read or delete object contents. +# +# @example Hiera: write-only (upload-only) access to buckets owned by existing $roles accounts +# role::storage::s3::write_only_roles: +# upload-only: +# access_key: ENC[PKCS7,...] +# email: upload-only@example.com +# buckets: +# - bucket: app-uploads +# owner: app +# - bucket: app2-uploads +# owner: app2 # # @param manage Whether to manage the S3 storage resources. Defaults to true. # @@ -24,14 +57,15 @@ # @groups network endpoint. # class role::storage::s3 ( - Stdlib::Fqdn $endpoint, - Stdlib::Unixpath $data_dir, - Stdlib::Unixpath $metadata_dir, - Stdlib::Unixpath $conf_dir, - Eit_types::Storage::S3 $roles, - Eit_types::Storage::S3::ReadOnly $read_only_roles, - Boolean $manage = true, - String $image = 'zenko/cloudserver:latest-7.10.19', + Stdlib::Fqdn $endpoint, + Stdlib::Unixpath $data_dir, + Stdlib::Unixpath $metadata_dir, + Stdlib::Unixpath $conf_dir, + Eit_types::Storage::S3 $roles, + Eit_types::Storage::S3::ReadOnly $read_only_roles, + Boolean $manage = true, + String $image = 'zenko/cloudserver:latest-7.10.19', + Eit_types::Storage::S3::WriteOnly $write_only_roles = {}, ) inherits role::storage { contain role::virtualization::docker