Refactor to use the crossplane.pythonic package.

Upgrade to use crossplane-sdk-python v0.9.0.
Implement being able to specify the full path to the Composite class.

Author: Patrick J. McNerthney

CloudOps.groovy

@@ -27,17 +27,17 @@ RUN \
     && /venv/build/bin/pip install hatch \
     && /venv/build/bin/hatch build -t wheel /whl
 
-# Create a fresh venv and install only the function wheel into it.
+# Create a fresh venv and install only the pythonic wheel into it.
 #RUN --mount=type=cache,target=/root/.cache/pip \
 RUN \
     python3 -m venv /venv/fn \
     && /venv/fn/bin/pip install /whl/*.whl
 
-# Copy the function venv to our runtime stage. It's important that the path be
+# Copy the pythonic venv to our runtime stage. It's important that the path be
 # the same as in the build stage, to avoid shebang paths and symlinks breaking. 
 FROM gcr.io/distroless/python3-debian12 AS image
 WORKDIR /
 USER nonroot:nonroot
 COPY --from=build --chown=nonroot:nonroot /venv/fn /venv/fn
 EXPOSE 9443
-ENTRYPOINT ["/venv/fn/bin/function"]
+ENTRYPOINT ["/venv/fn/bin/pythonic"]

Dockerfile

@@ -35,6 +35,8 @@ spec:
             vpc.spec.forProvider.cidrBlock = self.spec.cidr
             self.status.vpcId = vpc.status.atProvider.vpcId
 ```
+In addtion to an inline script, the python implementation can be specified
+as the complete path to a python class. See [Filing system Composites](#filing-system-composites).
 
 ## Examples
 
@@ -43,6 +45,17 @@ function-go-templating examples implemented using function-pythonic.
 The [eks-cluster](./examples/eks-cluster/composition.yaml) example is a good
 complex example creating the entire vpc structure needed for an EKS cluster.
 
+## Installing function-pythonic
+
+```yaml
+apiVersion: pkg.crossplane.io/v1
+kind: Function
+metadata:
+  name: function-pythonic
+spec:
+  package: ghcr.io/fortra/function-pythonic:v0.0.3
+```
+
 ## Managed Resource Dependencies
 
 function-pythonic automatically handles dependencies between managed resources.
@@ -117,8 +130,8 @@ The following functions are provided to create Protobuf structures:
 | Unknown | Create a new Protobuf unknown placeholder |
 | Yaml | Create a new Protobuf structure from a yaml string |
 | Json | Create a new Protobuf structure from a json string |
-| Base64Encode | Encode a string into base 64 |
-| Base64Decode | Decode a string from base 64 |
+| B64Encode | Encode a string into base 64 |
+| B64Decode | Decode a string from base 64 |
 
 The following items are supported in all the Protobuf Message wrapper classes: `bool`,
 `len`, `contains`, `iter`, `hash`, `==`, `str`, `format`
@@ -259,10 +272,73 @@ spec:
         self.status.composite = 'Hello, World!'
 ```
 
-## Installing Python Packages
+## Filing system Composites
+
+Composition Composite implementations can be coded in a stand alone python files
+by configuring the function-pythonic deployment with the code mounted into
+the package-runtime container, and then adding the mount point to the python
+path using the --python-path command line option. For example:
+```yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  namespace: crossplane-system
+  name: pythonic-composites
+data:
+  bucket.py: |
+    class BucketComposite(BaseComposite):
+        def compose(self):
+            self.resources.bucket.apiVersion = 's3.aws.upbound.io/v1beta2'
+            self.resources.bucket.kind = 'Bucket'
+            self.resources.bucket.spec.forProvider.region = 'us-east-1'  
+```
+```yaml
+apiVersion: pkg.crossplane.io/v1beta1
+kind: DeploymentRuntimeConfig
+metadata:
+  name: function-pythonic
+spec:
+  deploymentTemplate:
+    spec:
+      template:
+        spec:
+          containers:
+          - name: package-runtime
+            args:
+            - --debug
+            - --python-path
+            - /mnt/composites
+            volumeMounts:
+            - name: composites
+              mountPath: /mnt/composites
+          volumes:
+          - name: composites
+            configMap:
+              name: pythonic-composites
+```
+```yaml
+apiVersion: apiextensions.crossplane.io/v1
+kind: Composition
+metadata:
+  name: buckets.example.pythonic.io/v1
+spec:
+  compositeTypeRef:
+    apiVersion: example.pythonic.io/v1
+    kind: Bucket
+  mode: Pipeline
+  pipeline:
+  - step: create-bucket
+    functionRef:
+      name: function-pythonic
+    input:
+      apiVersion: pythonic.fn.fortra.com/v1alpha1
+      kind: Composite
+      composite: bucket.BucketComposite
+```
+## Install Additional Python Packages
 
 function-pythonic supports a `--pip-install` command line option which will run pip install
-with the configured pip install command. For example, the following DeploymentRuntimeConfig:
+with the configured pip install command. For example:
 ```yaml
 apiVersion: pkg.crossplane.io/v1beta1
 kind: DeploymentRuntimeConfig
@@ -280,3 +356,27 @@ spec:
             - --pip-install
             - --quiet aiobotocore==2.23.2
 ```
+
+## Enable Oversize Protos
+
+The Protobuf python package used by function-pythonic limits the depth of yaml
+elements and the total size of yaml parsed. This results in a limit of approximately
+30 levels of nested yaml fields. This check can be disabled using the `--allow-oversize-protos`
+command line option. For example:
+
+```yaml
+apiVersion: pkg.crossplane.io/v1beta1
+kind: DeploymentRuntimeConfig
+metadata:
+  name: function-pythonic
+spec:
+  deploymentTemplate:
+    spec:
+      template:
+        spec:
+          containers:
+          - name: package-runtime
+            args:
+            - --debug
+            - --allow-oversize-protos
+```

README.md

@@ -0,0 +1,17 @@
+import base64
+
+from .composite import BaseComposite
+from .protobuf import Map, List, Unknown, Yaml, Json
+B64Encode = lambda s: base64.b64encode(s.encode('utf-8')).decode('utf-8')
+B64Decode = lambda s: base64.b64decode(s.encode('utf-8')).decode('utf-8')
+
+__all__ = [
+    'BaseComposite',
+    'Map',
+    'List',
+    'Unknown',
+    'Yaml',
+    'Json',
+    'B64Encode',
+    'B64Decode',
+]

crossplane/pythonic/__init__.py

@@ -2,16 +2,16 @@
 import datetime
 from crossplane.function.proto.v1 import run_function_pb2 as fnv1
 
-import function.protobuf
+from . import protobuf
 
 
 _notset = object()
 
 
 class BaseComposite:
     def __init__(self, request, response, logger):
-        self.request = function.protobuf.Message(None, None, request.DESCRIPTOR, request, 'Function Request')
-        self.response = function.protobuf.Message(None, None, response.DESCRIPTOR, response)
+        self.request = protobuf.Message(None, None, request.DESCRIPTOR, request, 'Function Request')
+        self.response = protobuf.Message(None, None, response.DESCRIPTOR, response)
         self.logger = logger
         self.autoReady = True
         self.credentials = Credentials(self.request)
@@ -54,7 +54,7 @@ def ready(self):
     def ready(self, ready):
         if ready:
             ready = fnv1.Ready.READY_TRUE
-        elif ready == None or (isinstance(ready, function.protobuf.Values) and ready._isUnknown):
+        elif ready == None or (isinstance(ready, protobuf.Values) and ready._isUnknown):
             ready = fnv1.Ready.READY_UNSPECIFIED
         else:
             ready = fnv1.Ready.READY_FALSE
@@ -216,7 +216,7 @@ def ready(self):
     def ready(self, ready):
         if ready:
             ready = fnv1.Ready.READY_TRUE
-        elif ready == None or (isinstance(ready, function.protobuf.Values) and ready._isUnknown):
+        elif ready == None or (isinstance(ready, protobuf.Values) and ready._isUnknown):
             ready = fnv1.Ready.READY_UNSPECIFIED
         else:
             ready = fnv1.Ready.READY_FALSE
@@ -422,7 +422,7 @@ def claim(self, claim):
         if bool(self):
             if claim:
                 self._result.target = fnv1.Target.TARGET_COMPOSITE_AND_CLAIM
-            elif claim == None or (isinstance(claim, function.protobuf.Values) and claim._isUnknown):
+            elif claim == None or (isinstance(claim, protobuf.Values) and claim._isUnknown):
                 self._result.target = fnv1.Target.TARGET_UNSPECIFIED
             else:
                 self._result.target = fnv1.Target.TARGET_COMPOSITE
@@ -461,7 +461,7 @@ def __getitem__(self, type):
         return Condition(self, type)
 
 
-class Condition(function.protobuf.ProtobufValue):
+class Condition(protobuf.ProtobufValue):
     def __init__(self, conditions, type):
         self._conditions = conditions
         self.type = type
@@ -509,7 +509,7 @@ def status(self, status):
             condition.status = fnv1.Status.STATUS_CONDITION_TRUE
         elif status == None:
             condition.status = fnv1.Status.STATUS_CONDITION_UNKNOWN
-        elif isinstance(status, function.protobuf.Values) and status._isUnknown:
+        elif isinstance(status, protobuf.Values) and status._isUnknown:
             condition.status = fnv1.Status.STATUS_CONDITION_UNSPECIFIED
         else:
             condition.status = fnv1.Status.STATUS_CONDITION_FALSE
@@ -556,7 +556,7 @@ def claim(self, claim):
         condition = self._find_condition(True)
         if claim:
             condition.target = fnv1.Target.TARGET_COMPOSITE_AND_CLAIM
-        elif claim == None or (isinstance(claim, function.protobuf.Values) and claim._isUnknown):
+        elif claim == None or (isinstance(claim, protobuf.Values) and claim._isUnknown):
             condition.target = fnv1.Target.TARGET_UNSPECIFIED
         else:
             condition.target = fnv1.Target.TARGET_COMPOSITE

function/__version__.py crossplane/pythonic/__version__.pyfunction/__version__.py renamed to crossplane/pythonic/__version__.py

@@ -2,15 +2,25 @@
 
 import asyncio
 import base64
+import builtins
+import importlib
 import inspect
 
 import grpc
 import crossplane.function.logging
 import crossplane.function.response
 from crossplane.function.proto.v1 import run_function_pb2 as fnv1
 from crossplane.function.proto.v1 import run_function_pb2_grpc as grpcv1
-import function.composite
-import function.protobuf
+from .. import pythonic
+
+builtins.BaseComposite = pythonic.BaseComposite
+builtins.Map = pythonic.Map
+builtins.List = pythonic.List
+builtins.Unknown = pythonic.Unknown
+builtins.Yaml = pythonic.Yaml
+builtins.Json = pythonic.Json
+builtins.B64Encode = pythonic.B64Encode
+builtins.B64Decode = pythonic.B64Decode
 
 
 class FunctionRunner(grpcv1.FunctionRunnerService):
@@ -19,7 +29,7 @@ class FunctionRunner(grpcv1.FunctionRunnerService):
     def __init__(self):
         """Create a new FunctionRunner."""
         self.logger = crossplane.function.logging.get_logger()
-        self.modules = {}
+        self.clazzes = {}
 
     async def RunFunction(
         self, request: fnv1.RunFunctionRequest, _: grpc.aio.ServicerContext
@@ -52,23 +62,47 @@ async def RunFunction(
                 return response
             composite = input['composite']
 
-        module = self.modules.get(composite)
-        if not module:
-            module = Module()
-            try:
-                exec(composite, module.__dict__)
-            except Exception as e:
-                crossplane.function.response.fatal(response, f"Exec exception: {e}")
-                logger.exception('Exec exception')
+        clazz = self.clazzes.get(composite)
+        if not clazz:
+            if '\n' in composite:
+                module = Module()
+                try:
+                    exec(composite, module.__dict__)
+                except Exception as e:
+                    crossplane.function.response.fatal(response, f"Exec exception: {e}")
+                    logger.exception('Exec exception')
+                    return response
+                composite = ['<script>', 'Composite']
+            else:
+                composite = composite.rsplit('.', 1)
+                if len(composite) == 1:
+                    crossplane.function.response.fatal(response, f"Composite class name does not include module: {composite[0]}")
+                    logger.error(f"Composite class name does not include module: {composite[0]}")
+                    return response
+                try:
+                    module = importlib.import_module(composite[0])
+                except Exception as e:
+                    crossplane.function.response.fatal(response, f"Import module exception: {e}")
+                    logger.exception('Import module exception')
+                    return response
+            clazz = getattr(module, composite[1], None)
+            if not clazz:
+                crossplane.function.response.fatal(response, f"{composite[0]} did not define: {composite[1]}")
+                logger.error(f"{composite[0]} did not define: {composite[1]}")
+                return response
+            composite = '.'.join(composite)
+            if not inspect.isclass(clazz):
+                crossplane.function.response.fatal(response, f"{composite} is not a class")
+                logger.error(f"{composite} is not a class")
                 return response
-            if not hasattr(module, 'Composite') or not inspect.isclass(module.Composite):
-                crossplane.function.response.fatal(response, 'Function did not define "class Composite')
-                logger.error('Composite did not define "class Composite"')
+            if not issubclass(clazz, BaseComposite):
+                crossplane.function.response.fatal(response, f"{composite} is not a subclass of BaseComposite")
+                logger.error(f"{composite} is not a subclass of BaseComposite")
                 return response
-            self.modules[composite] = module
+            self.clazzes[composite] = clazz
 
         try:
-            composite = module.Composite(request, response, logger)
+            composite = clazz(request, response, logger)
         except Exception as e:
             crossplane.function.response.fatal(response, f"Instatiate exception: {e}")
             logger.exception('Instatiate exception')
@@ -101,12 +135,4 @@ async def RunFunction(
 
 
 class Module:
-    def __init__(self):
-        self.BaseComposite = function.composite.BaseComposite
-        self.Map = function.protobuf.Map
-        self.List = function.protobuf.List
-        self.Unknown = function.protobuf.Unknown
-        self.Yaml = function.protobuf.Yaml
-        self.Json = function.protobuf.Json
-        self.B64Encode = lambda s: base64.b64encode(s.encode('utf-8')).decode('utf-8')
-        self.B64Decode = lambda s: base64.b64decode(s.encode('utf-8')).decode('utf-8')
+    pass