Feat: PEN-0000 Move api-deploy, remove redundant inline schemas

.github/workflows/build.yml

@@ -25,9 +25,9 @@ jobs:
         python-version: ["3.9", "3.10"]
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v6
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v7
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies

.github/workflows/docker.yml

@@ -1,4 +1,4 @@
-name: Docker Hub
+name: Docker
 
 concurrency:
   cancel-in-progress: true
@@ -9,43 +9,47 @@ on:
     branches:
       - 'develop'
       - 'main'
-    tags: 
+    tags:
       - '*.*.*'
-
+
+permissions:
+  contents: read
+  packages: write
+
 jobs:
   build:
     runs-on: ubuntu-latest
 
-
     steps:
       -
-        name: Checkout 
-        uses: actions/checkout@v2
+        name: Checkout
+        uses: actions/checkout@v6
       -
-        name: Login to Docker Hub
-        uses: docker/login-action@v1
+        name: Login to GitHub Container Registry
+        uses: docker/login-action@v4
         with:
-          username: fabfuel
-          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v4
       -
         name: Build and push
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v7
         with:
           context: .
           file: ./Dockerfile
-          platforms: linux/amd64,linux/arm64
+          platforms: linux/amd64
           push: true
-          tags: fabfuel/api-deploy:${{ github.ref_name }}
+          tags: ghcr.io/packmatic/api-deploy:${{ github.ref_name }}
       -
         name: "Build and push (tag: latest)"
         if: github.ref == 'refs/heads/develop'
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v7
         with:
           context: .
           file: ./Dockerfile
-          platforms: linux/amd64,linux/arm64
+          platforms: linux/amd64
           push: true
-          tags: fabfuel/api-deploy:latest
+          tags: ghcr.io/packmatic/api-deploy:latest

README.md

@@ -1 +1,109 @@
-# API Deploy
+# API Deploy
+
+Compile an OpenAPI spec into an Amazon API Gateway definition and deploy it.
+
+Packmatic fork of [fabfuel/api-deploy](https://github.com/fabfuel/api-deploy). Pushing to
+`develop` publishes `ghcr.io/packmatic/api-deploy:develop`, which `packaging`'s API Gateway
+deploy uses.
+
+## Install
+
+```bash
+docker run --rm -v "$PWD":/workspace -w /workspace \
+  ghcr.io/packmatic/api-deploy:develop \
+  api compile <config> <source> <target>
+```
+
+## Usage
+
+```bash
+api compile <config> <source> <target>   # compile only
+api deploy <config> <api-id> <stage> <source>
+```
+
+## Configuration
+
+```yaml
+gateway:
+  integrationHost: 'https://${stageVariables.host}'
+  connectionId: '${stageVariables.connectionId}'
+  removeScopes: true          # strip OAuth scopes; API Gateway rejects them
+  removeDescriptions: true    # strip `description` from components/schemas
+  removeExamples: true        # strip `example` / `examples`
+flatten:
+  dedupExternalRefs: true     # share resolved external $refs instead of copying them
+headers:
+  request: [Authorization, Content-Type]
+cors:
+  origin: '*'
+static:
+  files: [api.v1.yml]
+strict:
+  enabled: true
+  overwriteRequired: true
+  blocklist: [_links, _meta]
+generator:
+  languages: [typescript]
+  output: src/openapi/types
+```
+
+`removeExamples` and `dedupExternalRefs` both default to `false`, so existing configs
+compile byte-for-byte as before.
+
+### `flatten.dedupExternalRefs`
+
+API Gateway's `put-rest-api` / `import-rest-api` body limit is
+[6 MiB](https://docs.aws.amazon.com/apigateway/latest/api/API_PutRestApi.html#API_PutRestApi_RequestBody).
+By default every external `$ref` is resolved into a **full inline copy** at each use site, and
+YAML anchors are disabled, so a spec that shares a handful of error responses across a few
+hundred endpoints inflates enormously — Packmatic's spec inlined ~1,100 copies of the same
+six error schemas.
+
+With `dedupExternalRefs: true`, each resolved external schema is registered once under
+`components/schemas` and every use site becomes an internal `$ref`. API Gateway resolves
+internal `#/components/schemas/*` refs, so the result imports unchanged.
+
+Scope: refs in a `schema:` position, and the payload schema of any resolved object carrying
+`content` (responses, request bodies). The response *wrapper* stays inline — only the payload
+schema is shared. Refs nested inside `properties` are still inlined, as are parameter refs,
+which API Gateway requires inline.
+
+Component names derive from the ref filename, forced to be alphanumeric and to start with a
+letter (`responses/500-server-error.yml` → `Response500ServerError`), and are uniquified
+against existing components. Schemas with identical bodies collapse into one component even
+when reached through differently spelled refs.
+
+Effect on Packmatic's packaging spec: **6,364,377 B → 4,214,757 B**, or 3,915,048 B with
+`removeExamples` as well. API Gateway model count drops from ~1,214 to ~128, since identical
+models are shared rather than duplicated per use site; per-method validation is unchanged.
+
+### `gateway.removeExamples`
+
+Strips `example` and `examples`. A schema *property* literally named `example` is preserved —
+only keyword positions are stripped — and `x-amazon-apigateway-integration` blocks are left
+untouched. Note this runs before the CORS processor, so CORS response-header examples it
+generates afterwards remain.
+
+Do not enable it in a config that also has a `generator` section: the TypeScript generator
+emits `Example:` JSDoc from these values, so the generated types would lose those comments.
+Keep it in the API-Gateway-only config.
+
+## Releasing
+
+Merging to `develop` builds and pushes `ghcr.io/packmatic/api-deploy:develop` (and `:latest`).
+Pushing a `x.y.z` tag publishes that tag too. Nothing to run by hand.
+
+## Development
+
+```bash
+pip install . -r requirements-test.txt
+pytest
+flake8 api_deploy
+```
+
+The functional tests resolve external `$ref`s against the live schemas at
+`api.packmatic.io` / `api-staging.packmatic.io`, which are served from the
+[api-types](https://github.com/Packmatic/api-types) repo. When api-types changes a shared
+schema, `tests/openapi/*_target.yml` goes stale and must be regenerated — api-types 1.11.0
+widened the `urn.yml` pattern to support two ids and added fields to the error responses,
+which is why those fixtures were refreshed.

api_deploy/__init__.py

@@ -1 +1 @@
-VERSION = '0.17.0'
+VERSION = '0.18.0'

api_deploy/config.py

@@ -14,6 +14,7 @@ def __init__(self, config_file: ConfigFile, file_path) -> None:
                 'response': [],
             },
             'strict': {},
+            'flatten': {},
             'gateway': {},
             'cors': {},
             'static': {
@@ -32,6 +33,10 @@ def __init__(self, config_file: ConfigFile, file_path) -> None:
         default_config['gateway'].setdefault('connection_id', config_file.get('gateway', {}).get('connectionId', ''))
         default_config['gateway'].setdefault('remove_scopes', config_file.get('gateway', {}).get('removeScopes', False))
         default_config['gateway'].setdefault('remove_descriptions', config_file.get('gateway', {}).get('removeDescriptions', False))
+        default_config['gateway'].setdefault('remove_examples', config_file.get('gateway', {}).get('removeExamples', False))
+
+        default_config['flatten'].setdefault('dedup_external_refs',
+                                             config_file.get('flatten', {}).get('dedupExternalRefs', False))
 
         default_config['cors'].setdefault('allow_origin', config_file.get('cors', {}).get('origin', '*'))
 

api_deploy/converters.py

@@ -1,4 +1,7 @@
+import re
 from copy import deepcopy
+
+import yaml
 from mergedeep import merge
 from requests import get, HTTPError
 
@@ -18,7 +21,7 @@ def __init__(self) -> None:
     def default(cls, config: Config):
         default_manager = cls()
         default_manager.register(StaticFileProcessor(config, **config['static']))
-        default_manager.register(FlattenProcessor(config))
+        default_manager.register(FlattenProcessor(config, **config['flatten']))
         default_manager.register(PassthroughProcessor(config, **config['headers']))
         default_manager.register(StrictProcessor(config, **config['strict']))
         default_manager.register(ApiGatewayProcessor(config, **config['gateway']))
@@ -40,11 +43,14 @@ def process(self, original_schema: Schema) -> Schema:
 
 class FlattenProcessor(AbstractProcessor):
 
-    def __init__(self, config: Config, **kwargs) -> None:
+    def __init__(self, config: Config, dedup_external_refs=False, **kwargs) -> None:
         super().__init__(config)
         self.base_url = None
         self.used_refs = set()
         self.external_schemas = {}
+        self.dedup_external_refs = dedup_external_refs
+        self.hoisted_names_by_fingerprint = {}
+        self.schemas_to_register = {}
 
     def process(self, source: Schema) -> Schema:
         target = deepcopy(source)
@@ -63,6 +69,12 @@ def process(self, source: Schema) -> Schema:
         self.replace_refs_dict(target['paths'], target)
         self.replace_refs_dict(target['paths'], target)
 
+        # Register hoisted schemas, resolving refs inside them until nothing new is hoisted
+        while self.schemas_to_register:
+            newly_hoisted, self.schemas_to_register = self.schemas_to_register, {}
+            target['components']['schemas'].update(newly_hoisted)
+            self.replace_refs_dict(target['components']['schemas'], target)
+
         try:
             del target['components']['parameters']
         except KeyError:
@@ -78,6 +90,8 @@ def process(self, source: Schema) -> Schema:
 
     def replace_refs_dict(self, node, schema, replace_ref=True, enforce_replace=False):
         if self.is_ref(node) and (replace_ref or enforce_replace or self.is_external_ref(node)):
+            if self.dedup_external_refs and self.is_external_ref(node) and not enforce_replace:
+                return self.hoist_external_ref(node, schema, is_schema_position=not replace_ref)
             return self.lookup_ref(node, schema)
         elif self.is_ref(node):
             self.used_refs.add(self.get_ref_model_name(node['$ref']))
@@ -106,6 +120,65 @@ def get_to_used_schemas(schemas: dict, used_refs: set):
                 used_schemas[model] = schemas[model]
         return used_schemas
 
+    def hoist_external_ref(self, ref: dict, schema: Schema, is_schema_position: bool):
+        """Resolve an external $ref once into components/schemas and reference it from every use site.
+
+        API Gateway resolves internal $refs, so this collapses thousands of duplicated inline copies.
+        """
+        ref_url = ref['$ref']
+        resolved = deepcopy(self.lookup_ref(ref, schema))
+
+        if not isinstance(resolved, dict):
+            return resolved
+
+        if is_schema_position:
+            return self.register_hoisted_schema(ref_url, resolved, schema)
+
+        # Response and request body objects: hoist only the payload schema, keep the small wrapper inline
+        if isinstance(resolved.get('content'), dict):
+            for media_type, media in resolved['content'].items():
+                if isinstance(media, dict) and isinstance(media.get('schema'), dict):
+                    media['schema'] = self.register_hoisted_schema(
+                        f'{ref_url}#{media_type}', media['schema'], schema
+                    )
+
+        return resolved
+
+    def register_hoisted_schema(self, ref_url: str, body: dict, schema: Schema):
+        # Identical schemas reached through differently spelled refs must collapse into one component
+        fingerprint = yaml.dump(body, sort_keys=True)
+        component_name = self.hoisted_names_by_fingerprint.get(fingerprint)
+
+        if not component_name:
+            component_name = self.build_component_name(ref_url, schema)
+            self.hoisted_names_by_fingerprint[fingerprint] = component_name
+            self.schemas_to_register[component_name] = body
+
+        self.used_refs.add(component_name)
+
+        return {'$ref': f'#/components/schemas/{component_name}'}
+
+    def build_component_name(self, ref_url: str, schema: Schema):
+        directory, _, file_name = ref_url.split('#')[0].rpartition('/')
+        base_name = self.to_pascal_case(re.sub(r'\.(ya?ml|json)$', '', file_name))
+
+        # API Gateway model names must be alphanumeric and cannot start with a digit
+        if not base_name[:1].isalpha():
+            base_name = self.to_pascal_case(directory.rpartition('/')[2].rstrip('s')) + base_name
+
+        taken = set(schema.get('components', {}).get('schemas', {})) | set(self.schemas_to_register)
+        component_name = base_name
+        suffix = 2
+        while component_name in taken:
+            component_name = f'{base_name}{suffix}'
+            suffix += 1
+
+        return component_name
+
+    @staticmethod
+    def to_pascal_case(value: str):
+        return ''.join(part[:1].upper() + part[1:] for part in re.split(r'[^A-Za-z0-9]+', value) if part)
+
     @staticmethod
     def get_ref_model_name(ref):
         return ref.split('/')[-1]
@@ -203,12 +276,14 @@ def merge_all_of(self, node, schema: Schema):
 
 
 class ApiGatewayProcessor(AbstractProcessor):
-    def __init__(self, config: Config, integration_host, connection_id, remove_scopes, remove_descriptions, **kwargs) -> None:
+    def __init__(self, config: Config, integration_host, connection_id, remove_scopes, remove_descriptions,
+                 remove_examples=False, **kwargs) -> None:
         super().__init__(config)
         self.integration_host = integration_host
         self.connection_id = connection_id
         self.remove_scopes = remove_scopes
         self.remove_descriptions = remove_descriptions
+        self.remove_examples = remove_examples
 
     def process(self, schema: Schema) -> Schema:
         for path in schema['paths']:
@@ -237,6 +312,11 @@ def process(self, schema: Schema) -> Schema:
             for model_name in schema['components'].get('schemas', {}):
                 self._remove_descriptions(schema['components']['schemas'][model_name])
 
+        # Examples are documentation only, API Gateway ignores them
+        if self.remove_examples:
+            self._remove_examples(schema['paths'])
+            self._remove_examples(schema['components'])
+
         # Replace all authorizers with API key type
         for authorizer in schema['components'].get('securitySchemes', {}):
             scheme = schema['components']['securitySchemes'][authorizer]
@@ -264,6 +344,23 @@ def _remove_descriptions(self, schema: object):
                         self._remove_descriptions(schema['items']['properties'][property_name])
 
 
+    def _remove_examples(self, node: object):
+        if isinstance(node, dict):
+            node.pop('example', None)
+            node.pop('examples', None)
+
+            for key, value in node.items():
+                # Never treat a schema property literally named "example" as a keyword
+                if key == 'properties' and isinstance(value, dict):
+                    for property_schema in value.values():
+                        self._remove_examples(property_schema)
+                elif key != 'x-amazon-apigateway-integration':
+                    self._remove_examples(value)
+
+        elif isinstance(node, list):
+            for item in node:
+                self._remove_examples(item)
+
     def _get_response_codes(self, schema, path, method):
         responses = {
             'default': {